2024-11-27 21:13:36,550 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-27 21:13:36,565 main DEBUG Took 0.013159 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 21:13:36,565 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 21:13:36,566 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 21:13:36,566 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 21:13:36,568 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:13:36,577 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 21:13:36,593 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:13:36,595 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:13:36,595 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:13:36,596 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:13:36,596 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:13:36,596 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:13:36,597 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:13:36,598 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:13:36,598 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:13:36,598 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:13:36,600 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:13:36,600 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:13:36,601 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:13:36,601 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:13:36,602 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:13:36,602 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:13:36,603 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:13:36,603 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:13:36,604 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:13:36,604 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:13:36,605 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:13:36,605 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:13:36,606 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:13:36,606 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 21:13:36,607 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:13:36,607 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 21:13:36,609 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 21:13:36,611 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 21:13:36,613 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 21:13:36,613 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 21:13:36,614 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 21:13:36,614 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 21:13:36,622 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 21:13:36,624 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 21:13:36,626 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 21:13:36,626 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 21:13:36,627 main DEBUG createAppenders(={Console}) 2024-11-27 21:13:36,627 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-27 21:13:36,628 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-27 21:13:36,628 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-27 21:13:36,628 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 21:13:36,629 main DEBUG OutputStream closed 2024-11-27 21:13:36,629 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 21:13:36,629 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 21:13:36,629 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-27 21:13:36,694 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 21:13:36,696 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 21:13:36,697 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 21:13:36,698 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 21:13:36,699 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 21:13:36,699 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 21:13:36,699 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 21:13:36,699 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 21:13:36,700 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 21:13:36,700 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 21:13:36,700 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 21:13:36,700 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 21:13:36,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 21:13:36,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 21:13:36,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 21:13:36,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 21:13:36,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 21:13:36,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 21:13:36,705 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 21:13:36,705 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-27 21:13:36,705 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 21:13:36,706 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-27T21:13:36,932 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609 2024-11-27 21:13:36,935 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 21:13:36,936 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T21:13:36,944 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-27T21:13:36,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T21:13:36,976 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/cluster_5e7b66de-1091-31c3-1f71-380834b464b9, deleteOnExit=true 2024-11-27T21:13:36,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T21:13:36,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/test.cache.data in system properties and HBase conf 2024-11-27T21:13:36,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T21:13:36,979 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/hadoop.log.dir in system properties and HBase conf 2024-11-27T21:13:36,979 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T21:13:36,980 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T21:13:36,980 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T21:13:37,076 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T21:13:37,162 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T21:13:37,166 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T21:13:37,166 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T21:13:37,166 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T21:13:37,167 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T21:13:37,167 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T21:13:37,168 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T21:13:37,168 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T21:13:37,169 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T21:13:37,169 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T21:13:37,169 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/nfs.dump.dir in system properties and HBase conf 2024-11-27T21:13:37,170 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/java.io.tmpdir in system properties and HBase conf 2024-11-27T21:13:37,170 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T21:13:37,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T21:13:37,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T21:13:38,785 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T21:13:38,862 INFO [Time-limited test {}] log.Log(170): Logging initialized @3082ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T21:13:38,938 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T21:13:39,001 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T21:13:39,021 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T21:13:39,022 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T21:13:39,023 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T21:13:39,036 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T21:13:39,039 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/hadoop.log.dir/,AVAILABLE} 2024-11-27T21:13:39,040 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T21:13:39,225 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/java.io.tmpdir/jetty-localhost-39807-hadoop-hdfs-3_4_1-tests_jar-_-any-12167331837410341888/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T21:13:39,238 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:39807} 2024-11-27T21:13:39,238 INFO [Time-limited test {}] server.Server(415): Started @3460ms 2024-11-27T21:13:39,650 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T21:13:39,658 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T21:13:39,659 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T21:13:39,659 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T21:13:39,659 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T21:13:39,660 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/hadoop.log.dir/,AVAILABLE} 2024-11-27T21:13:39,661 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T21:13:39,789 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/java.io.tmpdir/jetty-localhost-42853-hadoop-hdfs-3_4_1-tests_jar-_-any-16501566103443885895/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T21:13:39,790 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:42853} 2024-11-27T21:13:39,790 INFO [Time-limited test {}] server.Server(415): Started @4012ms 2024-11-27T21:13:39,851 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T21:13:40,325 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/cluster_5e7b66de-1091-31c3-1f71-380834b464b9/data/data1/current/BP-937257277-172.17.0.2-1732742017740/current, will proceed with Du for space computation calculation, 2024-11-27T21:13:40,325 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/cluster_5e7b66de-1091-31c3-1f71-380834b464b9/data/data2/current/BP-937257277-172.17.0.2-1732742017740/current, will proceed with Du for space computation calculation, 2024-11-27T21:13:40,369 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T21:13:40,422 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xea3cb5b0ea4af05a with lease ID 0x84eb2b939a566313: Processing first storage report for DS-cfe71069-6693-4eee-806f-86c4d3b61464 from datanode DatanodeRegistration(127.0.0.1:40567, datanodeUuid=f92784c2-4a78-4fdf-8fac-aaadf4e44198, infoPort=36787, infoSecurePort=0, ipcPort=35101, storageInfo=lv=-57;cid=testClusterID;nsid=1866928772;c=1732742017740) 2024-11-27T21:13:40,423 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xea3cb5b0ea4af05a with lease ID 0x84eb2b939a566313: from storage DS-cfe71069-6693-4eee-806f-86c4d3b61464 node DatanodeRegistration(127.0.0.1:40567, datanodeUuid=f92784c2-4a78-4fdf-8fac-aaadf4e44198, infoPort=36787, infoSecurePort=0, ipcPort=35101, storageInfo=lv=-57;cid=testClusterID;nsid=1866928772;c=1732742017740), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T21:13:40,424 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xea3cb5b0ea4af05a with lease ID 0x84eb2b939a566313: Processing first storage report for DS-9b83f74d-aaa3-4798-9490-ad308826a53e from datanode DatanodeRegistration(127.0.0.1:40567, datanodeUuid=f92784c2-4a78-4fdf-8fac-aaadf4e44198, infoPort=36787, infoSecurePort=0, ipcPort=35101, storageInfo=lv=-57;cid=testClusterID;nsid=1866928772;c=1732742017740) 2024-11-27T21:13:40,424 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xea3cb5b0ea4af05a with lease ID 0x84eb2b939a566313: from storage DS-9b83f74d-aaa3-4798-9490-ad308826a53e node DatanodeRegistration(127.0.0.1:40567, datanodeUuid=f92784c2-4a78-4fdf-8fac-aaadf4e44198, infoPort=36787, infoSecurePort=0, ipcPort=35101, storageInfo=lv=-57;cid=testClusterID;nsid=1866928772;c=1732742017740), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T21:13:40,507 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609 2024-11-27T21:13:40,588 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/cluster_5e7b66de-1091-31c3-1f71-380834b464b9/zookeeper_0, clientPort=60751, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/cluster_5e7b66de-1091-31c3-1f71-380834b464b9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/cluster_5e7b66de-1091-31c3-1f71-380834b464b9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T21:13:40,600 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60751 2024-11-27T21:13:40,616 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T21:13:40,621 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T21:13:40,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741825_1001 (size=7) 2024-11-27T21:13:41,292 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de with version=8 2024-11-27T21:13:41,292 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/hbase-staging 2024-11-27T21:13:41,387 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T21:13:41,660 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4585c39bed03:0 server-side Connection retries=45 2024-11-27T21:13:41,672 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T21:13:41,673 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T21:13:41,678 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T21:13:41,679 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T21:13:41,679 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T21:13:41,862 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T21:13:41,947 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T21:13:41,960 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T21:13:41,965 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T21:13:42,002 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2965 (auto-detected) 2024-11-27T21:13:42,003 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-27T21:13:42,028 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38503 2024-11-27T21:13:42,057 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38503 connecting to ZooKeeper ensemble=127.0.0.1:60751 2024-11-27T21:13:42,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:385030x0, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T21:13:42,110 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38503-0x1015b46e86a0000 connected 2024-11-27T21:13:42,145 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T21:13:42,148 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T21:13:42,159 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T21:13:42,163 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de, hbase.cluster.distributed=false 2024-11-27T21:13:42,185 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T21:13:42,189 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38503 2024-11-27T21:13:42,189 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38503 2024-11-27T21:13:42,190 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38503 2024-11-27T21:13:42,190 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38503 2024-11-27T21:13:42,190 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38503 2024-11-27T21:13:42,305 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4585c39bed03:0 server-side Connection retries=45 2024-11-27T21:13:42,307 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T21:13:42,307 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T21:13:42,307 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T21:13:42,307 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T21:13:42,308 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T21:13:42,311 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T21:13:42,313 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T21:13:42,314 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37381 2024-11-27T21:13:42,316 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37381 connecting to ZooKeeper ensemble=127.0.0.1:60751 2024-11-27T21:13:42,318 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T21:13:42,321 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T21:13:42,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:373810x0, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T21:13:42,332 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37381-0x1015b46e86a0001 connected 2024-11-27T21:13:42,332 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T21:13:42,337 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T21:13:42,346 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T21:13:42,348 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T21:13:42,353 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T21:13:42,354 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37381 2024-11-27T21:13:42,355 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37381 2024-11-27T21:13:42,355 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37381 2024-11-27T21:13:42,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37381 2024-11-27T21:13:42,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37381 2024-11-27T21:13:42,375 DEBUG [M:0;4585c39bed03:38503 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4585c39bed03:38503 2024-11-27T21:13:42,376 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4585c39bed03,38503,1732742021440 2024-11-27T21:13:42,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T21:13:42,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T21:13:42,384 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4585c39bed03,38503,1732742021440 2024-11-27T21:13:42,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:13:42,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T21:13:42,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:13:42,407 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T21:13:42,408 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4585c39bed03,38503,1732742021440 from backup master directory 2024-11-27T21:13:42,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T21:13:42,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4585c39bed03,38503,1732742021440 2024-11-27T21:13:42,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T21:13:42,412 WARN [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T21:13:42,412 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4585c39bed03,38503,1732742021440 2024-11-27T21:13:42,415 INFO [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T21:13:42,417 INFO [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T21:13:42,473 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/hbase.id] with ID: 74733cde-8d64-4f1f-94fd-09392088645c 2024-11-27T21:13:42,474 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/.tmp/hbase.id 2024-11-27T21:13:42,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741826_1002 (size=42) 2024-11-27T21:13:42,885 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/.tmp/hbase.id]:[hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/hbase.id] 2024-11-27T21:13:42,929 INFO [master/4585c39bed03:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T21:13:42,934 INFO [master/4585c39bed03:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T21:13:42,955 INFO [master/4585c39bed03:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-27T21:13:42,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:13:42,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:13:42,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741827_1003 (size=196) 2024-11-27T21:13:43,398 INFO [master/4585c39bed03:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:13:43,400 INFO [master/4585c39bed03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T21:13:43,417 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:43,422 INFO [master/4585c39bed03:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T21:13:43,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741828_1004 (size=1189) 2024-11-27T21:13:43,872 INFO [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store 2024-11-27T21:13:43,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741829_1005 (size=34) 2024-11-27T21:13:44,300 INFO [master/4585c39bed03:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T21:13:44,303 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:13:44,305 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T21:13:44,305 INFO [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T21:13:44,305 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T21:13:44,307 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T21:13:44,307 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T21:13:44,307 INFO [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T21:13:44,308 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732742024304Disabling compacts and flushes for region at 1732742024304Disabling writes for close at 1732742024307 (+3 ms)Writing region close event to WAL at 1732742024307Closed at 1732742024307 2024-11-27T21:13:44,312 WARN [master/4585c39bed03:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/.initializing 2024-11-27T21:13:44,312 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/WALs/4585c39bed03,38503,1732742021440 2024-11-27T21:13:44,322 INFO [master/4585c39bed03:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T21:13:44,344 INFO [master/4585c39bed03:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4585c39bed03%2C38503%2C1732742021440, suffix=, logDir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/WALs/4585c39bed03,38503,1732742021440, archiveDir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/oldWALs, maxLogs=10 2024-11-27T21:13:44,380 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/WALs/4585c39bed03,38503,1732742021440/4585c39bed03%2C38503%2C1732742021440.1732742024351, exclude list is [], retry=0 2024-11-27T21:13:44,405 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40567,DS-cfe71069-6693-4eee-806f-86c4d3b61464,DISK] 2024-11-27T21:13:44,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T21:13:44,461 INFO [master/4585c39bed03:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/WALs/4585c39bed03,38503,1732742021440/4585c39bed03%2C38503%2C1732742021440.1732742024351 2024-11-27T21:13:44,463 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36787:36787)] 2024-11-27T21:13:44,463 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:13:44,464 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:13:44,469 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:13:44,470 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:13:44,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:13:44,571 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T21:13:44,581 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:44,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:13:44,588 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:13:44,598 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T21:13:44,600 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:44,602 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:13:44,602 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:13:44,609 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T21:13:44,609 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:44,611 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:13:44,613 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:13:44,619 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T21:13:44,620 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:44,621 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:13:44,622 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:13:44,627 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:13:44,629 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:13:44,637 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:13:44,639 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:13:44,645 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T21:13:44,651 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T21:13:44,669 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T21:13:44,672 INFO [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70019546, jitterRate=0.043372541666030884}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T21:13:44,679 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732742024489Initializing all the Stores at 1732742024492 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732742024493 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742024494 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742024494Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742024494Cleaning up temporary data from old regions at 1732742024640 (+146 ms)Region opened successfully at 1732742024679 (+39 ms) 2024-11-27T21:13:44,681 INFO [master/4585c39bed03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T21:13:44,729 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72150504, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4585c39bed03/172.17.0.2:0 2024-11-27T21:13:44,782 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T21:13:44,798 INFO [master/4585c39bed03:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T21:13:44,799 INFO [master/4585c39bed03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T21:13:44,803 INFO [master/4585c39bed03:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T21:13:44,805 INFO [master/4585c39bed03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-27T21:13:44,812 INFO [master/4585c39bed03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-27T21:13:44,812 INFO [master/4585c39bed03:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T21:13:44,850 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T21:13:44,864 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T21:13:44,867 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T21:13:44,870 INFO [master/4585c39bed03:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T21:13:44,873 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T21:13:44,874 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T21:13:44,878 INFO [master/4585c39bed03:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T21:13:44,883 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T21:13:44,884 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T21:13:44,886 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T21:13:44,889 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T21:13:44,912 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T21:13:44,914 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T21:13:44,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T21:13:44,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T21:13:44,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:13:44,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:13:44,925 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4585c39bed03,38503,1732742021440, sessionid=0x1015b46e86a0000, setting cluster-up flag (Was=false) 2024-11-27T21:13:44,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:13:44,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:13:44,956 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T21:13:44,958 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4585c39bed03,38503,1732742021440 2024-11-27T21:13:44,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:13:44,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:13:44,974 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T21:13:44,976 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4585c39bed03,38503,1732742021440 2024-11-27T21:13:44,984 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T21:13:45,063 INFO [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(746): ClusterId : 74733cde-8d64-4f1f-94fd-09392088645c 2024-11-27T21:13:45,066 DEBUG [RS:0;4585c39bed03:37381 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T21:13:45,072 DEBUG [RS:0;4585c39bed03:37381 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T21:13:45,072 DEBUG [RS:0;4585c39bed03:37381 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T21:13:45,073 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T21:13:45,077 DEBUG [RS:0;4585c39bed03:37381 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T21:13:45,077 DEBUG [RS:0;4585c39bed03:37381 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@394c81eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4585c39bed03/172.17.0.2:0 2024-11-27T21:13:45,086 INFO [master/4585c39bed03:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T21:13:45,094 DEBUG [RS:0;4585c39bed03:37381 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4585c39bed03:37381 2024-11-27T21:13:45,094 INFO [master/4585c39bed03:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T21:13:45,098 INFO [RS:0;4585c39bed03:37381 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T21:13:45,098 INFO [RS:0;4585c39bed03:37381 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T21:13:45,098 DEBUG [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T21:13:45,101 INFO [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(2659): reportForDuty to master=4585c39bed03,38503,1732742021440 with port=37381, startcode=1732742022265 2024-11-27T21:13:45,101 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4585c39bed03,38503,1732742021440 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T21:13:45,110 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4585c39bed03:0, corePoolSize=5, maxPoolSize=5 2024-11-27T21:13:45,110 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4585c39bed03:0, corePoolSize=5, maxPoolSize=5 2024-11-27T21:13:45,111 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4585c39bed03:0, corePoolSize=5, maxPoolSize=5 2024-11-27T21:13:45,111 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4585c39bed03:0, corePoolSize=5, maxPoolSize=5 2024-11-27T21:13:45,111 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4585c39bed03:0, corePoolSize=10, maxPoolSize=10 2024-11-27T21:13:45,111 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:13:45,112 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4585c39bed03:0, corePoolSize=2, maxPoolSize=2 2024-11-27T21:13:45,112 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:13:45,115 DEBUG [RS:0;4585c39bed03:37381 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T21:13:45,125 INFO [master/4585c39bed03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732742055125 2024-11-27T21:13:45,125 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T21:13:45,125 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T21:13:45,126 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T21:13:45,127 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T21:13:45,132 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T21:13:45,133 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T21:13:45,133 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T21:13:45,133 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T21:13:45,134 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:45,134 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T21:13:45,135 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:45,141 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T21:13:45,143 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T21:13:45,143 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T21:13:45,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741831_1007 (size=1321) 2024-11-27T21:13:45,150 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T21:13:45,150 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de 2024-11-27T21:13:45,151 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T21:13:45,152 INFO [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T21:13:45,156 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4585c39bed03:0:becomeActiveMaster-HFileCleaner.large.0-1732742025153,5,FailOnTimeoutGroup] 2024-11-27T21:13:45,157 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4585c39bed03:0:becomeActiveMaster-HFileCleaner.small.0-1732742025157,5,FailOnTimeoutGroup] 2024-11-27T21:13:45,157 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:45,157 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T21:13:45,160 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:45,160 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:45,161 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33985, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T21:13:45,169 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38503 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4585c39bed03,37381,1732742022265 2024-11-27T21:13:45,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741832_1008 (size=32) 2024-11-27T21:13:45,172 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38503 {}] master.ServerManager(517): Registering regionserver=4585c39bed03,37381,1732742022265 2024-11-27T21:13:45,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:13:45,176 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T21:13:45,179 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T21:13:45,179 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:45,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:13:45,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T21:13:45,184 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T21:13:45,184 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:45,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:13:45,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T21:13:45,189 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T21:13:45,189 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:45,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:13:45,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T21:13:45,191 DEBUG [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de 2024-11-27T21:13:45,191 DEBUG [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42399 2024-11-27T21:13:45,191 DEBUG [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T21:13:45,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T21:13:45,197 DEBUG [RS:0;4585c39bed03:37381 {}] zookeeper.ZKUtil(111): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4585c39bed03,37381,1732742022265 2024-11-27T21:13:45,197 WARN [RS:0;4585c39bed03:37381 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T21:13:45,197 INFO [RS:0;4585c39bed03:37381 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T21:13:45,198 DEBUG [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/WALs/4585c39bed03,37381,1732742022265 2024-11-27T21:13:45,199 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T21:13:45,200 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:45,201 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4585c39bed03,37381,1732742022265] 2024-11-27T21:13:45,202 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:13:45,202 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T21:13:45,206 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740 2024-11-27T21:13:45,207 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740 2024-11-27T21:13:45,211 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T21:13:45,211 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T21:13:45,213 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:13:45,216 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T21:13:45,220 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T21:13:45,221 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59952140, jitterRate=-0.10664349794387817}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:13:45,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732742025173Initializing all the Stores at 1732742025175 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732742025175Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732742025175Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742025176 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732742025176Cleaning up temporary data from old regions at 1732742025211 (+35 ms)Region opened successfully at 1732742025225 (+14 ms) 2024-11-27T21:13:45,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T21:13:45,225 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T21:13:45,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T21:13:45,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T21:13:45,226 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T21:13:45,229 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T21:13:45,229 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732742025225Disabling compacts and flushes for region at 1732742025225Disabling writes for close at 1732742025226 (+1 ms)Writing region close event to WAL at 1732742025229 (+3 ms)Closed at 1732742025229 2024-11-27T21:13:45,234 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T21:13:45,234 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T21:13:45,238 INFO [RS:0;4585c39bed03:37381 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T21:13:45,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T21:13:45,256 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T21:13:45,257 INFO [RS:0;4585c39bed03:37381 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T21:13:45,259 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T21:13:45,265 INFO [RS:0;4585c39bed03:37381 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T21:13:45,265 INFO [RS:0;4585c39bed03:37381 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:45,267 INFO [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T21:13:45,273 INFO [RS:0;4585c39bed03:37381 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T21:13:45,274 INFO [RS:0;4585c39bed03:37381 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:45,275 DEBUG [RS:0;4585c39bed03:37381 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:13:45,275 DEBUG [RS:0;4585c39bed03:37381 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:13:45,275 DEBUG [RS:0;4585c39bed03:37381 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:13:45,275 DEBUG [RS:0;4585c39bed03:37381 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:13:45,275 DEBUG [RS:0;4585c39bed03:37381 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:13:45,276 DEBUG [RS:0;4585c39bed03:37381 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4585c39bed03:0, corePoolSize=2, maxPoolSize=2 2024-11-27T21:13:45,276 DEBUG [RS:0;4585c39bed03:37381 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:13:45,276 DEBUG [RS:0;4585c39bed03:37381 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:13:45,276 DEBUG [RS:0;4585c39bed03:37381 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:13:45,276 DEBUG [RS:0;4585c39bed03:37381 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:13:45,276 DEBUG [RS:0;4585c39bed03:37381 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:13:45,276 DEBUG [RS:0;4585c39bed03:37381 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4585c39bed03:0, corePoolSize=1, maxPoolSize=1 2024-11-27T21:13:45,276 DEBUG [RS:0;4585c39bed03:37381 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4585c39bed03:0, corePoolSize=3, maxPoolSize=3 2024-11-27T21:13:45,277 DEBUG [RS:0;4585c39bed03:37381 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0, corePoolSize=3, maxPoolSize=3 2024-11-27T21:13:45,277 INFO [RS:0;4585c39bed03:37381 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:45,278 INFO [RS:0;4585c39bed03:37381 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:45,278 INFO [RS:0;4585c39bed03:37381 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:45,278 INFO [RS:0;4585c39bed03:37381 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:45,278 INFO [RS:0;4585c39bed03:37381 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:45,278 INFO [RS:0;4585c39bed03:37381 {}] hbase.ChoreService(168): Chore ScheduledChore name=4585c39bed03,37381,1732742022265-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T21:13:45,298 INFO [RS:0;4585c39bed03:37381 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T21:13:45,300 INFO [RS:0;4585c39bed03:37381 {}] hbase.ChoreService(168): Chore ScheduledChore name=4585c39bed03,37381,1732742022265-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:45,300 INFO [RS:0;4585c39bed03:37381 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:45,300 INFO [RS:0;4585c39bed03:37381 {}] regionserver.Replication(171): 4585c39bed03,37381,1732742022265 started 2024-11-27T21:13:45,318 INFO [RS:0;4585c39bed03:37381 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:45,319 INFO [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1482): Serving as 4585c39bed03,37381,1732742022265, RpcServer on 4585c39bed03/172.17.0.2:37381, sessionid=0x1015b46e86a0001 2024-11-27T21:13:45,319 DEBUG [RS:0;4585c39bed03:37381 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T21:13:45,320 DEBUG [RS:0;4585c39bed03:37381 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4585c39bed03,37381,1732742022265 2024-11-27T21:13:45,320 DEBUG [RS:0;4585c39bed03:37381 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4585c39bed03,37381,1732742022265' 2024-11-27T21:13:45,320 DEBUG [RS:0;4585c39bed03:37381 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T21:13:45,321 DEBUG [RS:0;4585c39bed03:37381 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T21:13:45,322 DEBUG [RS:0;4585c39bed03:37381 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T21:13:45,322 DEBUG [RS:0;4585c39bed03:37381 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T21:13:45,322 DEBUG [RS:0;4585c39bed03:37381 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4585c39bed03,37381,1732742022265 2024-11-27T21:13:45,322 DEBUG [RS:0;4585c39bed03:37381 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4585c39bed03,37381,1732742022265' 2024-11-27T21:13:45,322 DEBUG [RS:0;4585c39bed03:37381 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T21:13:45,323 DEBUG [RS:0;4585c39bed03:37381 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T21:13:45,324 DEBUG [RS:0;4585c39bed03:37381 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T21:13:45,324 INFO [RS:0;4585c39bed03:37381 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T21:13:45,324 INFO [RS:0;4585c39bed03:37381 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T21:13:45,411 WARN [4585c39bed03:38503 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T21:13:45,429 INFO [RS:0;4585c39bed03:37381 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T21:13:45,432 INFO [RS:0;4585c39bed03:37381 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4585c39bed03%2C37381%2C1732742022265, suffix=, logDir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/WALs/4585c39bed03,37381,1732742022265, archiveDir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/oldWALs, maxLogs=32 2024-11-27T21:13:45,449 DEBUG [RS:0;4585c39bed03:37381 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/WALs/4585c39bed03,37381,1732742022265/4585c39bed03%2C37381%2C1732742022265.1732742025435, exclude list is [], retry=0 2024-11-27T21:13:45,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40567,DS-cfe71069-6693-4eee-806f-86c4d3b61464,DISK] 2024-11-27T21:13:45,458 INFO [RS:0;4585c39bed03:37381 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/WALs/4585c39bed03,37381,1732742022265/4585c39bed03%2C37381%2C1732742022265.1732742025435 2024-11-27T21:13:45,458 DEBUG [RS:0;4585c39bed03:37381 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36787:36787)] 2024-11-27T21:13:45,663 DEBUG [4585c39bed03:38503 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-27T21:13:45,676 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:13:45,685 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4585c39bed03,37381,1732742022265, state=OPENING 2024-11-27T21:13:45,697 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T21:13:45,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:13:45,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:13:45,700 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T21:13:45,700 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T21:13:45,701 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T21:13:45,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:13:45,882 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T21:13:45,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52197, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T21:13:45,899 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T21:13:45,900 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T21:13:45,900 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T21:13:45,904 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4585c39bed03%2C37381%2C1732742022265.meta, suffix=.meta, logDir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/WALs/4585c39bed03,37381,1732742022265, archiveDir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/oldWALs, maxLogs=32 2024-11-27T21:13:45,926 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/WALs/4585c39bed03,37381,1732742022265/4585c39bed03%2C37381%2C1732742022265.meta.1732742025906.meta, exclude list is [], retry=0 2024-11-27T21:13:45,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40567,DS-cfe71069-6693-4eee-806f-86c4d3b61464,DISK] 2024-11-27T21:13:45,939 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/WALs/4585c39bed03,37381,1732742022265/4585c39bed03%2C37381%2C1732742022265.meta.1732742025906.meta 2024-11-27T21:13:45,939 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36787:36787)] 2024-11-27T21:13:45,940 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:13:45,942 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T21:13:45,946 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T21:13:45,952 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T21:13:45,957 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T21:13:45,957 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:13:45,958 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T21:13:45,958 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T21:13:45,962 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T21:13:45,964 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T21:13:45,964 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:45,965 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:13:45,965 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T21:13:45,967 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T21:13:45,967 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:45,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:13:45,969 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T21:13:45,970 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T21:13:45,970 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:45,971 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:13:45,972 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T21:13:45,973 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T21:13:45,973 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:45,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T21:13:45,974 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T21:13:45,976 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740 2024-11-27T21:13:45,979 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740 2024-11-27T21:13:45,982 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T21:13:45,982 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T21:13:45,984 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:13:45,988 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T21:13:45,990 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67233457, jitterRate=0.001856580376625061}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:13:45,991 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T21:13:45,993 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732742025958Writing region info on filesystem at 1732742025959 (+1 ms)Initializing all the Stores at 1732742025961 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732742025961Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732742025961Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742025961Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732742025962 (+1 ms)Cleaning up temporary data from old regions at 1732742025983 (+21 ms)Running coprocessor post-open hooks at 1732742025991 (+8 ms)Region opened successfully at 1732742025993 (+2 ms) 2024-11-27T21:13:46,002 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732742025872 2024-11-27T21:13:46,019 DEBUG [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T21:13:46,020 INFO [RS_OPEN_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T21:13:46,024 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:13:46,027 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4585c39bed03,37381,1732742022265, state=OPEN 2024-11-27T21:13:46,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T21:13:46,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T21:13:46,035 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T21:13:46,036 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:46,038 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T21:13:46,043 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T21:13:46,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4585c39bed03,37381,1732742022265 in 333 msec 2024-11-27T21:13:46,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T21:13:46,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 802 msec 2024-11-27T21:13:46,053 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T21:13:46,053 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T21:13:46,075 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:13:46,076 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:13:46,097 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:13:46,101 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45433, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:13:46,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1020 sec 2024-11-27T21:13:46,123 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732742026123, completionTime=-1 2024-11-27T21:13:46,126 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-27T21:13:46,126 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T21:13:46,157 INFO [master/4585c39bed03:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-27T21:13:46,157 INFO [master/4585c39bed03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732742086157 2024-11-27T21:13:46,157 INFO [master/4585c39bed03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732742146157 2024-11-27T21:13:46,157 INFO [master/4585c39bed03:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-11-27T21:13:46,159 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4585c39bed03,38503,1732742021440-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:46,159 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4585c39bed03,38503,1732742021440-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:46,160 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4585c39bed03,38503,1732742021440-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:46,161 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4585c39bed03:38503, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:46,162 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:46,162 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:46,168 DEBUG [master/4585c39bed03:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T21:13:46,191 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.779sec 2024-11-27T21:13:46,192 INFO [master/4585c39bed03:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T21:13:46,193 INFO [master/4585c39bed03:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T21:13:46,194 INFO [master/4585c39bed03:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T21:13:46,195 INFO [master/4585c39bed03:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T21:13:46,195 INFO [master/4585c39bed03:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T21:13:46,196 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4585c39bed03,38503,1732742021440-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T21:13:46,196 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4585c39bed03,38503,1732742021440-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T21:13:46,205 DEBUG [master/4585c39bed03:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T21:13:46,206 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T21:13:46,206 INFO [master/4585c39bed03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4585c39bed03,38503,1732742021440-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T21:13:46,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:46,276 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T21:13:46,276 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T21:13:46,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:13:46,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:13:46,290 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:13:46,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:13:46,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:13:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@531238f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:13:46,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:13:46,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:13:46,301 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:13:46,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:46,305 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:13:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:13:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:13:46,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:13:46,337 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4585c39bed03,38503,1732742021440 2024-11-27T21:13:46,350 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=311, ProcessCount=11, AvailableMemoryMB=9052 2024-11-27T21:13:46,359 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T21:13:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 4585c39bed03,38503,1732742021440 2024-11-27T21:13:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@571ad612 2024-11-27T21:13:46,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T21:13:46,371 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T21:13:46,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:13:46,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:13:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T21:13:46,387 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T21:13:46,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-27T21:13:46,389 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:46,391 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T21:13:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T21:13:46,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741835_1011 (size=1061) 2024-11-27T21:13:46,404 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de 2024-11-27T21:13:46,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741836_1012 (size=53) 2024-11-27T21:13:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T21:13:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T21:13:46,814 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:13:46,814 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cc764913a0f35f9f71e6ee97aeccf332, disabling compactions & flushes 2024-11-27T21:13:46,814 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:46,814 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:46,814 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. after waiting 0 ms 2024-11-27T21:13:46,814 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:46,814 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:46,815 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cc764913a0f35f9f71e6ee97aeccf332: Waiting for close lock at 1732742026814Disabling compacts and flushes for region at 1732742026814Disabling writes for close at 1732742026814Writing region close event to WAL at 1732742026814Closed at 1732742026814 2024-11-27T21:13:46,817 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T21:13:46,823 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732742026817"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732742026817"}]},"ts":"1732742026817"} 2024-11-27T21:13:46,833 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T21:13:46,835 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T21:13:46,837 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742026835"}]},"ts":"1732742026835"} 2024-11-27T21:13:46,842 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T21:13:46,844 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc764913a0f35f9f71e6ee97aeccf332, ASSIGN}] 2024-11-27T21:13:46,846 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc764913a0f35f9f71e6ee97aeccf332, ASSIGN 2024-11-27T21:13:46,848 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc764913a0f35f9f71e6ee97aeccf332, ASSIGN; state=OFFLINE, location=4585c39bed03,37381,1732742022265; forceNewPlan=false, retain=false 2024-11-27T21:13:46,999 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cc764913a0f35f9f71e6ee97aeccf332, regionState=OPENING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:13:47,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc764913a0f35f9f71e6ee97aeccf332, ASSIGN because future has completed 2024-11-27T21:13:47,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:13:47,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T21:13:47,168 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:47,169 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:13:47,170 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:47,170 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:13:47,170 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:47,170 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:47,180 INFO [StoreOpener-cc764913a0f35f9f71e6ee97aeccf332-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:47,184 INFO [StoreOpener-cc764913a0f35f9f71e6ee97aeccf332-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:13:47,185 INFO [StoreOpener-cc764913a0f35f9f71e6ee97aeccf332-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc764913a0f35f9f71e6ee97aeccf332 columnFamilyName A 2024-11-27T21:13:47,185 DEBUG [StoreOpener-cc764913a0f35f9f71e6ee97aeccf332-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:47,186 INFO [StoreOpener-cc764913a0f35f9f71e6ee97aeccf332-1 {}] regionserver.HStore(327): Store=cc764913a0f35f9f71e6ee97aeccf332/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:13:47,186 INFO [StoreOpener-cc764913a0f35f9f71e6ee97aeccf332-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:47,188 INFO [StoreOpener-cc764913a0f35f9f71e6ee97aeccf332-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:13:47,189 INFO [StoreOpener-cc764913a0f35f9f71e6ee97aeccf332-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc764913a0f35f9f71e6ee97aeccf332 columnFamilyName B 2024-11-27T21:13:47,189 DEBUG [StoreOpener-cc764913a0f35f9f71e6ee97aeccf332-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:47,190 INFO [StoreOpener-cc764913a0f35f9f71e6ee97aeccf332-1 {}] regionserver.HStore(327): Store=cc764913a0f35f9f71e6ee97aeccf332/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:13:47,190 INFO [StoreOpener-cc764913a0f35f9f71e6ee97aeccf332-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:47,192 INFO [StoreOpener-cc764913a0f35f9f71e6ee97aeccf332-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:13:47,193 INFO [StoreOpener-cc764913a0f35f9f71e6ee97aeccf332-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc764913a0f35f9f71e6ee97aeccf332 columnFamilyName C 2024-11-27T21:13:47,193 DEBUG [StoreOpener-cc764913a0f35f9f71e6ee97aeccf332-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:13:47,194 INFO [StoreOpener-cc764913a0f35f9f71e6ee97aeccf332-1 {}] regionserver.HStore(327): Store=cc764913a0f35f9f71e6ee97aeccf332/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:13:47,195 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:47,195 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:47,196 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:47,197 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:47,199 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:47,199 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:47,201 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:13:47,204 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:47,208 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T21:13:47,209 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened cc764913a0f35f9f71e6ee97aeccf332; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65321873, jitterRate=-0.026628240942955017}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:13:47,210 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:47,211 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for cc764913a0f35f9f71e6ee97aeccf332: Running coprocessor pre-open hook at 1732742027171Writing region info on filesystem at 1732742027171Initializing all the Stores at 1732742027173 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742027173Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742027180 (+7 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742027180Cleaning up temporary data from old regions at 1732742027200 (+20 ms)Running coprocessor post-open hooks at 1732742027210 (+10 ms)Region opened successfully at 1732742027211 (+1 ms) 2024-11-27T21:13:47,214 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., pid=6, masterSystemTime=1732742027161 2024-11-27T21:13:47,219 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:47,220 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:47,221 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cc764913a0f35f9f71e6ee97aeccf332, regionState=OPEN, openSeqNum=2, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:13:47,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:13:47,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T21:13:47,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 in 224 msec 2024-11-27T21:13:47,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T21:13:47,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc764913a0f35f9f71e6ee97aeccf332, ASSIGN in 393 msec 2024-11-27T21:13:47,246 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T21:13:47,246 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742027246"}]},"ts":"1732742027246"} 2024-11-27T21:13:47,251 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T21:13:47,253 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T21:13:47,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 874 msec 2024-11-27T21:13:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T21:13:47,531 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:13:47,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2c994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,539 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:13:47,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:13:47,541 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:13:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:13:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:13:47,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ecb3032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:13:47,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:13:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:13:47,546 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:13:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3922c1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,549 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:13:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:13:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:13:47,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:13:47,559 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T21:13:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 4585c39bed03,38503,1732742021440 2024-11-27T21:13:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@21f850a5 2024-11-27T21:13:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T21:13:47,566 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T21:13:47,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:13:47,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:13:47,573 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:13:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:13:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:13:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f5a6b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:13:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:13:47,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:13:47,577 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:13:47,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b383f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:13:47,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:13:47,581 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:13:47,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:13:47,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:13:47,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef1bd73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:13:47,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:13:47,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:13:47,585 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:13:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af2c0a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:13:47,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:13:47,594 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:13:47,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:13:47,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:13:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc8436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:13:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:13:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:13:47,603 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:13:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,606 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:13:47,606 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:13:47,611 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:13:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:13:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:13:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553a6f47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:13:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:13:47,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:13:47,620 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:13:47,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eefee7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,624 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:13:47,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:13:47,628 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:13:47,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:13:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:13:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c2308e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:13:47,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:13:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:13:47,632 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:13:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f776e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,636 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:13:47,637 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:13:47,638 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:13:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:13:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:13:47,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6e4493, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:13:47,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:13:47,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:13:47,643 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:13:47,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b5a398e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,649 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:13:47,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:13:47,652 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:13:47,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:13:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:13:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e2fecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:13:47,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:13:47,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:13:47,657 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:13:47,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79379aab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,660 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:13:47,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:13:47,662 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:13:47,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:13:47,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:13:47,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b3b865c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:13:47,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:13:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:13:47,665 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:13:47,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13210d53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:13:47,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:13:47,670 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:13:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:13:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:13:47,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6602a8c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:13:47,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:13:47,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:13:47,673 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:13:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7933c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:13:47,682 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:13:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:13:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:13:47,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:13:47,687 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:13:47,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:13:47,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:13:47,691 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:13:47,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:13:47,695 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:13:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:13:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:13:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:13:47,699 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:13:47,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:13:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:13:47,702 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:13:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:13:47,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:13:47,706 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:13:47,706 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:13:47,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:13:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:13:47,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:13:47,713 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:13:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:13:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:13:47,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:13:47,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:13:47,729 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:13:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:13:47,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:13:47,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:13:47,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:13:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-27T21:13:47,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:13:47,743 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:13:47,745 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:13:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:13:47,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:13:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:13:47,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:13:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T21:13:47,750 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:13:47,751 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:13:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:13:47,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:13:47,783 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:13:47,784 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:13:47,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:13:47,790 DEBUG [Thread-154 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:13:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:47,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:13:47,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:47,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T21:13:47,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:47,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:47,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:47,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:47,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:13:47,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:47,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:47,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:47,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:47,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:48,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/1c04f06cab2f46528d8846d928318fb4 is 50, key is test_row_0/A:col10/1732742027819/Put/seqid=0 2024-11-27T21:13:48,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741837_1013 (size=9657) 2024-11-27T21:13:48,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T21:13:48,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/1c04f06cab2f46528d8846d928318fb4 2024-11-27T21:13:48,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732742028010, max timestamp=1732742028083Num uniques 859; 2024-11-27T21:13:48,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-27T21:13:48,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742028010, max timestamp=1732742028080Num uniques 859; 2024-11-27T21:13:48,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742028010, max timestamp=1732742028078Num uniques 859; 2024-11-27T21:13:48,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:13:48,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:13:48,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:13:48,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:48,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:48,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:48,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/3b8bbb2b61224970886f6a851253dcc0 is 50, key is test_row_0/B:col10/1732742027819/Put/seqid=0 2024-11-27T21:13:48,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:48,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:48,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741838_1014 (size=9657) 2024-11-27T21:13:48,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/3b8bbb2b61224970886f6a851253dcc0 2024-11-27T21:13:48,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742088215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:48,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742088224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:48,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742088225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:48,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742088230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:48,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742088233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:48,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/21bba86033084555919a384735430167 is 50, key is test_row_0/C:col10/1732742027819/Put/seqid=0 2024-11-27T21:13:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:48,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741839_1015 (size=9657) 2024-11-27T21:13:48,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:13:48,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:48,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:48,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:48,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:48,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T21:13:48,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742088380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:48,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:48,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742088381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742088380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:48,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742088382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742088382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:48,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:48,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:48,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:13:48,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:48,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:48,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:48,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:48,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:48,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742088600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:48,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:48,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742088601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:48,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742088601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:48,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:48,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742088610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:48,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:48,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742088610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:48,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:48,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:48,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:13:48,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:48,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:48,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:48,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:48,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:48,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/21bba86033084555919a384735430167 2024-11-27T21:13:48,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/1c04f06cab2f46528d8846d928318fb4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1c04f06cab2f46528d8846d928318fb4 2024-11-27T21:13:48,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1c04f06cab2f46528d8846d928318fb4, entries=100, sequenceid=12, filesize=9.4 K 2024-11-27T21:13:48,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/3b8bbb2b61224970886f6a851253dcc0 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/3b8bbb2b61224970886f6a851253dcc0 2024-11-27T21:13:48,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/3b8bbb2b61224970886f6a851253dcc0, entries=100, sequenceid=12, filesize=9.4 K 2024-11-27T21:13:48,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/21bba86033084555919a384735430167 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/21bba86033084555919a384735430167 2024-11-27T21:13:48,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/21bba86033084555919a384735430167, entries=100, sequenceid=12, filesize=9.4 K 2024-11-27T21:13:48,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=133.78 KB/136995 for cc764913a0f35f9f71e6ee97aeccf332 in 951ms, sequenceid=12, compaction requested=false 2024-11-27T21:13:48,779 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T21:13:48,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:48,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T21:13:48,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:48,832 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-11-27T21:13:48,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:48,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:48,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:48,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:48,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:48,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:48,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/ce9216f0286d4cbc8361c7247b540557 is 50, key is test_row_0/A:col10/1732742028206/Put/seqid=0 2024-11-27T21:13:48,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741840_1016 (size=12001) 2024-11-27T21:13:48,888 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/ce9216f0286d4cbc8361c7247b540557 2024-11-27T21:13:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T21:13:48,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/4b312f79fffb439d905092483e7b12d9 is 50, key is test_row_0/B:col10/1732742028206/Put/seqid=0 2024-11-27T21:13:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:48,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:48,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741841_1017 (size=12001) 2024-11-27T21:13:48,955 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/4b312f79fffb439d905092483e7b12d9 2024-11-27T21:13:48,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/60534b66d9464a33b4e9919226f7eca9 is 50, key is test_row_0/C:col10/1732742028206/Put/seqid=0 2024-11-27T21:13:49,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742088979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741842_1018 (size=12001) 2024-11-27T21:13:49,022 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/60534b66d9464a33b4e9919226f7eca9 2024-11-27T21:13:49,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742088996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742088997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/ce9216f0286d4cbc8361c7247b540557 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce9216f0286d4cbc8361c7247b540557 2024-11-27T21:13:49,060 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce9216f0286d4cbc8361c7247b540557, entries=150, sequenceid=49, filesize=11.7 K 2024-11-27T21:13:49,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/4b312f79fffb439d905092483e7b12d9 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/4b312f79fffb439d905092483e7b12d9 2024-11-27T21:13:49,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742089004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742089007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,076 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/4b312f79fffb439d905092483e7b12d9, entries=150, sequenceid=49, filesize=11.7 K 2024-11-27T21:13:49,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/60534b66d9464a33b4e9919226f7eca9 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/60534b66d9464a33b4e9919226f7eca9 2024-11-27T21:13:49,090 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/60534b66d9464a33b4e9919226f7eca9, entries=150, sequenceid=49, filesize=11.7 K 2024-11-27T21:13:49,092 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=53.67 KB/54960 for cc764913a0f35f9f71e6ee97aeccf332 in 259ms, sequenceid=49, compaction requested=false 2024-11-27T21:13:49,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:49,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:49,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-27T21:13:49,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-27T21:13:49,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T21:13:49,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3430 sec 2024-11-27T21:13:49,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.3640 sec 2024-11-27T21:13:49,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:49,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-27T21:13:49,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:49,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:49,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:49,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:49,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:49,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:49,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/ce4974811b354f49a1b418b0b08a8c64 is 50, key is test_row_0/A:col10/1732742029138/Put/seqid=0 2024-11-27T21:13:49,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741843_1019 (size=12001) 2024-11-27T21:13:49,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/ce4974811b354f49a1b418b0b08a8c64 2024-11-27T21:13:49,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/6645439617a944ceac2d9d0ec9c785ab is 50, key is test_row_0/B:col10/1732742029138/Put/seqid=0 2024-11-27T21:13:49,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741844_1020 (size=12001) 2024-11-27T21:13:49,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/6645439617a944ceac2d9d0ec9c785ab 2024-11-27T21:13:49,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/731990192ab84e3dbe89c6be62d151f3 is 50, key is test_row_0/C:col10/1732742029138/Put/seqid=0 2024-11-27T21:13:49,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742089320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742089323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742089325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742089327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742089332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741845_1021 (size=12001) 2024-11-27T21:13:49,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/731990192ab84e3dbe89c6be62d151f3 2024-11-27T21:13:49,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/ce4974811b354f49a1b418b0b08a8c64 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce4974811b354f49a1b418b0b08a8c64 2024-11-27T21:13:49,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce4974811b354f49a1b418b0b08a8c64, entries=150, sequenceid=62, filesize=11.7 K 2024-11-27T21:13:49,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/6645439617a944ceac2d9d0ec9c785ab as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6645439617a944ceac2d9d0ec9c785ab 2024-11-27T21:13:49,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6645439617a944ceac2d9d0ec9c785ab, entries=150, sequenceid=62, filesize=11.7 K 2024-11-27T21:13:49,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/731990192ab84e3dbe89c6be62d151f3 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/731990192ab84e3dbe89c6be62d151f3 2024-11-27T21:13:49,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/731990192ab84e3dbe89c6be62d151f3, entries=150, sequenceid=62, filesize=11.7 K 2024-11-27T21:13:49,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cc764913a0f35f9f71e6ee97aeccf332 in 281ms, sequenceid=62, compaction requested=true 2024-11-27T21:13:49,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:49,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:13:49,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:49,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:13:49,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:49,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:13:49,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:13:49,457 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:13:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:49,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-27T21:13:49,460 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:13:49,462 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:13:49,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742029320, max timestamp=1732742029457Num uniques 859; 2024-11-27T21:13:49,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:13:49,464 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:13:49,464 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/B is initiating minor compaction (all files) 2024-11-27T21:13:49,464 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/A is initiating minor compaction (all files) 2024-11-27T21:13:49,464 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/B in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:49,465 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/A in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:49,465 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1c04f06cab2f46528d8846d928318fb4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce9216f0286d4cbc8361c7247b540557, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce4974811b354f49a1b418b0b08a8c64] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=32.9 K 2024-11-27T21:13:49,465 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/3b8bbb2b61224970886f6a851253dcc0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/4b312f79fffb439d905092483e7b12d9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6645439617a944ceac2d9d0ec9c785ab] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=32.9 K 2024-11-27T21:13:49,468 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c04f06cab2f46528d8846d928318fb4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732742027798 2024-11-27T21:13:49,469 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b8bbb2b61224970886f6a851253dcc0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732742027798 2024-11-27T21:13:49,470 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce9216f0286d4cbc8361c7247b540557, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732742028206 2024-11-27T21:13:49,471 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b312f79fffb439d905092483e7b12d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732742028206 2024-11-27T21:13:49,471 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce4974811b354f49a1b418b0b08a8c64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732742028999 2024-11-27T21:13:49,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742029320, max timestamp=1732742029457Num uniques 859; 2024-11-27T21:13:49,472 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 6645439617a944ceac2d9d0ec9c785ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732742028999 2024-11-27T21:13:49,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:49,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:13:49,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:49,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:49,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:49,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:49,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:49,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/8035c3b5af6440cfb39d24189f8cdfe4 is 50, key is test_row_0/A:col10/1732742029323/Put/seqid=0 2024-11-27T21:13:49,519 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#A#compaction#10 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:49,521 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/1c3c4aef223d4aba908295f573fded10 is 50, key is test_row_0/A:col10/1732742029138/Put/seqid=0 2024-11-27T21:13:49,539 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#B#compaction#11 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:49,541 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/49e954ca358a4ced9c544666b45ba16d is 50, key is test_row_0/B:col10/1732742029138/Put/seqid=0 2024-11-27T21:13:49,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742089538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742089556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741846_1022 (size=16681) 2024-11-27T21:13:49,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/8035c3b5af6440cfb39d24189f8cdfe4 2024-11-27T21:13:49,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742089558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742089560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742029533, max timestamp=1732742029557Num uniques 859; 2024-11-27T21:13:49,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:13:49,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741847_1023 (size=12104) 2024-11-27T21:13:49,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742029533, max timestamp=1732742029557Num uniques 859; 2024-11-27T21:13:49,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:13:49,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741848_1024 (size=12104) 2024-11-27T21:13:49,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/6f648534c10942fc939d1c97e27885fc is 50, key is test_row_0/B:col10/1732742029323/Put/seqid=0 2024-11-27T21:13:49,622 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/1c3c4aef223d4aba908295f573fded10 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1c3c4aef223d4aba908295f573fded10 2024-11-27T21:13:49,628 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/49e954ca358a4ced9c544666b45ba16d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/49e954ca358a4ced9c544666b45ba16d 2024-11-27T21:13:49,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732742029533, max timestamp=1732742029557Num uniques 859; 2024-11-27T21:13:49,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-27T21:13:49,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741849_1025 (size=12001) 2024-11-27T21:13:49,655 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/A of cc764913a0f35f9f71e6ee97aeccf332 into 1c3c4aef223d4aba908295f573fded10(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:49,655 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:49,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/6f648534c10942fc939d1c97e27885fc 2024-11-27T21:13:49,658 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/A, priority=13, startTime=1732742029437; duration=0sec 2024-11-27T21:13:49,658 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:13:49,658 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:A 2024-11-27T21:13:49,660 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/B of cc764913a0f35f9f71e6ee97aeccf332 into 49e954ca358a4ced9c544666b45ba16d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:49,660 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:49,660 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/B, priority=13, startTime=1732742029454; duration=0sec 2024-11-27T21:13:49,661 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:49,661 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:B 2024-11-27T21:13:49,661 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:13:49,666 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:13:49,667 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/C is initiating minor compaction (all files) 2024-11-27T21:13:49,667 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/C in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:49,667 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/21bba86033084555919a384735430167, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/60534b66d9464a33b4e9919226f7eca9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/731990192ab84e3dbe89c6be62d151f3] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=32.9 K 2024-11-27T21:13:49,669 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21bba86033084555919a384735430167, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732742027798 2024-11-27T21:13:49,672 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60534b66d9464a33b4e9919226f7eca9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732742028206 2024-11-27T21:13:49,673 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 731990192ab84e3dbe89c6be62d151f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732742028999 2024-11-27T21:13:49,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/7bf0d1fb7cee4169a618d60d5faefc64 is 50, key is test_row_0/C:col10/1732742029323/Put/seqid=0 2024-11-27T21:13:49,700 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#C#compaction#14 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:49,701 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/de2b65f65b994843a4fa6501325c1adb is 50, key is test_row_0/C:col10/1732742029138/Put/seqid=0 2024-11-27T21:13:49,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741850_1026 (size=12001) 2024-11-27T21:13:49,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/7bf0d1fb7cee4169a618d60d5faefc64 2024-11-27T21:13:49,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/8035c3b5af6440cfb39d24189f8cdfe4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/8035c3b5af6440cfb39d24189f8cdfe4 2024-11-27T21:13:49,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742089730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741851_1027 (size=12104) 2024-11-27T21:13:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742089732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742089740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/8035c3b5af6440cfb39d24189f8cdfe4, entries=250, sequenceid=83, filesize=16.3 K 2024-11-27T21:13:49,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/6f648534c10942fc939d1c97e27885fc as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6f648534c10942fc939d1c97e27885fc 2024-11-27T21:13:49,763 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/de2b65f65b994843a4fa6501325c1adb as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/de2b65f65b994843a4fa6501325c1adb 2024-11-27T21:13:49,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742089744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742089745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6f648534c10942fc939d1c97e27885fc, entries=150, sequenceid=83, filesize=11.7 K 2024-11-27T21:13:49,781 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/C of cc764913a0f35f9f71e6ee97aeccf332 into de2b65f65b994843a4fa6501325c1adb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:49,781 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:49,781 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/C, priority=13, startTime=1732742029454; duration=0sec 2024-11-27T21:13:49,781 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:49,781 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:C 2024-11-27T21:13:49,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/7bf0d1fb7cee4169a618d60d5faefc64 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7bf0d1fb7cee4169a618d60d5faefc64 2024-11-27T21:13:49,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7bf0d1fb7cee4169a618d60d5faefc64, entries=150, sequenceid=83, filesize=11.7 K 2024-11-27T21:13:49,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~62.62 KB/64120, heapSize ~174.28 KB/178464, currentSize=127.08 KB/130125 for cc764913a0f35f9f71e6ee97aeccf332 in 336ms, sequenceid=83, compaction requested=false 2024-11-27T21:13:49,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:49,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-11-27T21:13:49,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:49,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:49,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:49,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:49,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:49,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:49,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/34870c2ef1424b2a91e8283fb4a14f39 is 50, key is test_row_0/A:col10/1732742029740/Put/seqid=0 2024-11-27T21:13:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T21:13:49,900 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:13:49,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:13:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-27T21:13:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T21:13:49,904 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:13:49,906 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:13:49,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:13:49,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742089892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742089893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742089897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742089900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:49,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742089901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:49,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:49,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741852_1028 (size=14341) 2024-11-27T21:13:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T21:13:50,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742090019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:50,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742090020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:50,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742090020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:50,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742090020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:50,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:50,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742090020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:50,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:50,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T21:13:50,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:50,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:50,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:50,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T21:13:50,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T21:13:50,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:50,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:50,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:50,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:50,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742090230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:50,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:50,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:50,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742090239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:50,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:50,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:50,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742090239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742090240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:50,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:50,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742090240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:50,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/34870c2ef1424b2a91e8283fb4a14f39 2024-11-27T21:13:50,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/5f11ead49d104f5fbfebc4f8621a04ad is 50, key is test_row_0/B:col10/1732742029740/Put/seqid=0 2024-11-27T21:13:50,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741853_1029 (size=12001) 2024-11-27T21:13:50,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/5f11ead49d104f5fbfebc4f8621a04ad 2024-11-27T21:13:50,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T21:13:50,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:50,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:50,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:50,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/ca6c2fabebb945e0a2b5a4af130a9b1c is 50, key is test_row_0/C:col10/1732742029740/Put/seqid=0 2024-11-27T21:13:50,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741854_1030 (size=12001) 2024-11-27T21:13:50,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T21:13:50,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T21:13:50,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:50,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:50,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:50,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:50,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742090539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:50,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:50,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:50,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742090551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:50,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:50,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742090550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:50,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:50,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742090552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:50,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:50,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742090552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:50,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:50,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:50,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T21:13:50,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:50,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:50,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:50,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/ca6c2fabebb945e0a2b5a4af130a9b1c 2024-11-27T21:13:50,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/34870c2ef1424b2a91e8283fb4a14f39 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/34870c2ef1424b2a91e8283fb4a14f39 2024-11-27T21:13:50,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T21:13:50,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/34870c2ef1424b2a91e8283fb4a14f39, entries=200, sequenceid=123, filesize=14.0 K 2024-11-27T21:13:50,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:50,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:50,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:50,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:50,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/5f11ead49d104f5fbfebc4f8621a04ad as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/5f11ead49d104f5fbfebc4f8621a04ad 2024-11-27T21:13:50,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/5f11ead49d104f5fbfebc4f8621a04ad, entries=150, sequenceid=123, filesize=11.7 K 2024-11-27T21:13:50,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/ca6c2fabebb945e0a2b5a4af130a9b1c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/ca6c2fabebb945e0a2b5a4af130a9b1c 2024-11-27T21:13:50,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/ca6c2fabebb945e0a2b5a4af130a9b1c, entries=150, sequenceid=123, filesize=11.7 K 2024-11-27T21:13:50,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=46.96 KB/48090 for cc764913a0f35f9f71e6ee97aeccf332 in 1028ms, sequenceid=123, compaction requested=true 2024-11-27T21:13:50,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:50,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:13:50,895 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:13:50,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:50,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:13:50,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:50,896 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:13:50,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:13:50,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:13:50,898 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:13:50,898 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43126 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:13:50,898 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/B is initiating minor compaction (all files) 2024-11-27T21:13:50,898 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/B in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:50,898 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/A is initiating minor compaction (all files) 2024-11-27T21:13:50,898 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/A in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:50,898 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/49e954ca358a4ced9c544666b45ba16d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6f648534c10942fc939d1c97e27885fc, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/5f11ead49d104f5fbfebc4f8621a04ad] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=35.3 K 2024-11-27T21:13:50,898 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1c3c4aef223d4aba908295f573fded10, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/8035c3b5af6440cfb39d24189f8cdfe4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/34870c2ef1424b2a91e8283fb4a14f39] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=42.1 K 2024-11-27T21:13:50,899 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 49e954ca358a4ced9c544666b45ba16d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732742028999 2024-11-27T21:13:50,899 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c3c4aef223d4aba908295f573fded10, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732742028999 2024-11-27T21:13:50,900 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f648534c10942fc939d1c97e27885fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732742029320 2024-11-27T21:13:50,900 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8035c3b5af6440cfb39d24189f8cdfe4, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732742029320 2024-11-27T21:13:50,901 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34870c2ef1424b2a91e8283fb4a14f39, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732742029733 2024-11-27T21:13:50,901 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f11ead49d104f5fbfebc4f8621a04ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732742029738 2024-11-27T21:13:50,928 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#A#compaction#18 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:50,929 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/2242659bb7c24f24998025fa07f70520 is 50, key is test_row_0/A:col10/1732742029740/Put/seqid=0 2024-11-27T21:13:50,931 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:50,932 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/bed736206d0e404d968be73f2c1932a5 is 50, key is test_row_0/B:col10/1732742029740/Put/seqid=0 2024-11-27T21:13:50,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741856_1032 (size=12207) 2024-11-27T21:13:50,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741855_1031 (size=12207) 2024-11-27T21:13:50,965 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/2242659bb7c24f24998025fa07f70520 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/2242659bb7c24f24998025fa07f70520 2024-11-27T21:13:50,979 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/A of cc764913a0f35f9f71e6ee97aeccf332 into 2242659bb7c24f24998025fa07f70520(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:50,980 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:50,980 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/A, priority=13, startTime=1732742030895; duration=0sec 2024-11-27T21:13:50,981 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:13:50,981 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:A 2024-11-27T21:13:50,981 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:13:50,983 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:13:50,983 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/C is initiating minor compaction (all files) 2024-11-27T21:13:50,983 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/C in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:50,983 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/de2b65f65b994843a4fa6501325c1adb, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7bf0d1fb7cee4169a618d60d5faefc64, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/ca6c2fabebb945e0a2b5a4af130a9b1c] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=35.3 K 2024-11-27T21:13:50,987 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting de2b65f65b994843a4fa6501325c1adb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732742028999 2024-11-27T21:13:50,989 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bf0d1fb7cee4169a618d60d5faefc64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732742029320 2024-11-27T21:13:50,990 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca6c2fabebb945e0a2b5a4af130a9b1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732742029738 2024-11-27T21:13:51,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T21:13:51,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:51,010 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:13:51,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:51,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:51,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:51,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:51,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:51,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:51,017 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#C#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:51,018 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/758588e2f15c445d87fee027a5d575c3 is 50, key is test_row_0/C:col10/1732742029740/Put/seqid=0 2024-11-27T21:13:51,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/997261afb33c42e6b11f8742f2c47a2e is 50, key is test_row_0/A:col10/1732742029892/Put/seqid=0 2024-11-27T21:13:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T21:13:51,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741858_1034 (size=12151) 2024-11-27T21:13:51,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741857_1033 (size=12207) 2024-11-27T21:13:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:51,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:51,068 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/758588e2f15c445d87fee027a5d575c3 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/758588e2f15c445d87fee027a5d575c3 2024-11-27T21:13:51,085 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/C of cc764913a0f35f9f71e6ee97aeccf332 into 758588e2f15c445d87fee027a5d575c3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:51,085 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:51,085 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/C, priority=13, startTime=1732742030896; duration=0sec 2024-11-27T21:13:51,086 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:51,086 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:C 2024-11-27T21:13:51,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742031097, max timestamp=1732742031111Num uniques 859; 2024-11-27T21:13:51,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742031097, max timestamp=1732742031111Num uniques 859; 2024-11-27T21:13:51,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:13:51,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:13:51,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742031097, max timestamp=1732742031110Num uniques 859; 2024-11-27T21:13:51,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:13:51,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742091155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:51,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742091163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:51,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742091170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:51,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742091174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:51,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742091174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:51,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:51,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742091279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:51,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:51,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742091290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:51,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742091279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:51,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:51,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742091291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:51,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:51,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742091290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:51,365 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/bed736206d0e404d968be73f2c1932a5 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/bed736206d0e404d968be73f2c1932a5 2024-11-27T21:13:51,384 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/B of cc764913a0f35f9f71e6ee97aeccf332 into bed736206d0e404d968be73f2c1932a5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:51,385 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:51,385 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/B, priority=13, startTime=1732742030896; duration=0sec 2024-11-27T21:13:51,385 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:51,385 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:B 2024-11-27T21:13:51,447 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/997261afb33c42e6b11f8742f2c47a2e 2024-11-27T21:13:51,473 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T21:13:51,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/71a726554bbe48bea347ccfe538bacac is 50, key is test_row_0/B:col10/1732742029892/Put/seqid=0 2024-11-27T21:13:51,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742091499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:51,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742091499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742091502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742091501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:51,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742091511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:51,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:51,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:51,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741859_1035 (size=12151) 2024-11-27T21:13:51,528 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/71a726554bbe48bea347ccfe538bacac 2024-11-27T21:13:51,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/f56027a8cfa346dab16aef7f346a0c67 is 50, key is test_row_0/C:col10/1732742029892/Put/seqid=0 2024-11-27T21:13:51,585 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T21:13:51,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741860_1036 (size=12151) 2024-11-27T21:13:51,601 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/f56027a8cfa346dab16aef7f346a0c67 2024-11-27T21:13:51,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/997261afb33c42e6b11f8742f2c47a2e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/997261afb33c42e6b11f8742f2c47a2e 2024-11-27T21:13:51,633 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/997261afb33c42e6b11f8742f2c47a2e, entries=150, sequenceid=134, filesize=11.9 K 2024-11-27T21:13:51,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/71a726554bbe48bea347ccfe538bacac as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/71a726554bbe48bea347ccfe538bacac 2024-11-27T21:13:51,647 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/71a726554bbe48bea347ccfe538bacac, entries=150, sequenceid=134, filesize=11.9 K 2024-11-27T21:13:51,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/f56027a8cfa346dab16aef7f346a0c67 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/f56027a8cfa346dab16aef7f346a0c67 2024-11-27T21:13:51,662 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/f56027a8cfa346dab16aef7f346a0c67, entries=150, sequenceid=134, filesize=11.9 K 2024-11-27T21:13:51,664 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for cc764913a0f35f9f71e6ee97aeccf332 in 654ms, sequenceid=134, compaction requested=false 2024-11-27T21:13:51,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:51,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:51,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-27T21:13:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-27T21:13:51,671 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-27T21:13:51,671 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7610 sec 2024-11-27T21:13:51,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.7710 sec 2024-11-27T21:13:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:51,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-27T21:13:51,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732742031160, max timestamp=1732742031811Num uniques 1068; 2024-11-27T21:13:51,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732742031160, max timestamp=1732742031811Num uniques 1068; 2024-11-27T21:13:51,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-27T21:13:51,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732742031160, max timestamp=1732742031811Num uniques 1068; 2024-11-27T21:13:51,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-27T21:13:51,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-27T21:13:51,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:51,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:51,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:51,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:51,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:51,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:51,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/c862038fa7ff49f6a670d2ac436f0447 is 50, key is test_row_0/A:col10/1732742031813/Put/seqid=0 2024-11-27T21:13:51,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741861_1037 (size=16931) 2024-11-27T21:13:51,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/c862038fa7ff49f6a670d2ac436f0447 2024-11-27T21:13:51,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/85e174d7564b4885829ab4492d03d2b7 is 50, key is test_row_0/B:col10/1732742031813/Put/seqid=0 2024-11-27T21:13:51,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742031851, max timestamp=1732742031868Num uniques 859; 2024-11-27T21:13:51,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:13:51,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742031851, max timestamp=1732742031868Num uniques 859; 2024-11-27T21:13:51,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:13:51,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742031851, max timestamp=1732742031868Num uniques 859; 2024-11-27T21:13:51,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:13:51,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741862_1038 (size=12151) 2024-11-27T21:13:51,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/85e174d7564b4885829ab4492d03d2b7 2024-11-27T21:13:51,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/95cf6d43045643e180cc060b0be50db8 is 50, key is test_row_0/C:col10/1732742031813/Put/seqid=0 2024-11-27T21:13:51,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=418, cellSize=19138, totalHeapSize=43712, min timestamp=1732742031895, max timestamp=1732742031925Num uniques 1018; 2024-11-27T21:13:51,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=19138, suffix heap size=133840,new segment heap size=43712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=418 2024-11-27T21:13:51,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732742031908, max timestamp=1732742031925Num uniques 1018; 2024-11-27T21:13:51,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-27T21:13:51,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732742031908, max timestamp=1732742031925Num uniques 1018; 2024-11-27T21:13:51,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-27T21:13:51,940 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T21:13:51,941 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T21:13:51,944 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T21:13:51,944 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T21:13:51,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741863_1039 (size=12151) 2024-11-27T21:13:51,944 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T21:13:51,944 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T21:13:51,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T21:13:51,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-27T21:13:51,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/95cf6d43045643e180cc060b0be50db8 2024-11-27T21:13:51,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/c862038fa7ff49f6a670d2ac436f0447 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c862038fa7ff49f6a670d2ac436f0447 2024-11-27T21:13:51,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c862038fa7ff49f6a670d2ac436f0447, entries=250, sequenceid=176, filesize=16.5 K 2024-11-27T21:13:51,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/85e174d7564b4885829ab4492d03d2b7 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/85e174d7564b4885829ab4492d03d2b7 2024-11-27T21:13:51,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/85e174d7564b4885829ab4492d03d2b7, entries=150, sequenceid=176, filesize=11.9 K 2024-11-27T21:13:51,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/95cf6d43045643e180cc060b0be50db8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/95cf6d43045643e180cc060b0be50db8 2024-11-27T21:13:51,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/95cf6d43045643e180cc060b0be50db8, entries=150, sequenceid=176, filesize=11.9 K 2024-11-27T21:13:52,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~105.19 KB/107712, currentSize=103.51 KB/105994 for cc764913a0f35f9f71e6ee97aeccf332 in 207ms, sequenceid=176, compaction requested=true 2024-11-27T21:13:52,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:52,021 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:13:52,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:13:52,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:52,023 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:13:52,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:13:52,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:52,024 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41289 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:13:52,024 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/A is initiating minor compaction (all files) 2024-11-27T21:13:52,024 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/A in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:52,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:13:52,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:13:52,024 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/2242659bb7c24f24998025fa07f70520, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/997261afb33c42e6b11f8742f2c47a2e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c862038fa7ff49f6a670d2ac436f0447] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=40.3 K 2024-11-27T21:13:52,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=123.64 KB heapSize=346.11 KB 2024-11-27T21:13:52,026 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:13:52,026 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 2242659bb7c24f24998025fa07f70520, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732742029738 2024-11-27T21:13:52,026 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/B is initiating minor compaction (all files) 2024-11-27T21:13:52,026 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/B in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:52,026 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/bed736206d0e404d968be73f2c1932a5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/71a726554bbe48bea347ccfe538bacac, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/85e174d7564b4885829ab4492d03d2b7] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=35.7 K 2024-11-27T21:13:52,028 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting bed736206d0e404d968be73f2c1932a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732742029738 2024-11-27T21:13:52,029 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 997261afb33c42e6b11f8742f2c47a2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732742029882 2024-11-27T21:13:52,029 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71a726554bbe48bea347ccfe538bacac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732742029882 2024-11-27T21:13:52,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:52,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:52,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:52,030 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting c862038fa7ff49f6a670d2ac436f0447, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732742031162 2024-11-27T21:13:52,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:52,030 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85e174d7564b4885829ab4492d03d2b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732742031169 2024-11-27T21:13:52,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:52,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T21:13:52,052 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#B#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:52,053 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/174ae1b327d84464adbb8254db9f5ffa is 50, key is test_row_0/B:col10/1732742031813/Put/seqid=0 2024-11-27T21:13:52,053 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:13:52,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:13:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-27T21:13:52,057 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:13:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T21:13:52,060 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:13:52,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:13:52,061 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:52,062 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/d9aaf39b25f9440a9aa9c5e5ada4dfe8 is 50, key is test_row_0/A:col10/1732742031813/Put/seqid=0 2024-11-27T21:13:52,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/398e722f967b4653a235f2e136dbf85f is 50, key is test_row_0/A:col10/1732742032021/Put/seqid=0 2024-11-27T21:13:52,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742092070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742092071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742092078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742092080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742092082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741865_1041 (size=12459) 2024-11-27T21:13:52,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741864_1040 (size=12459) 2024-11-27T21:13:52,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741866_1042 (size=14541) 2024-11-27T21:13:52,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/398e722f967b4653a235f2e136dbf85f 2024-11-27T21:13:52,136 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/d9aaf39b25f9440a9aa9c5e5ada4dfe8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/d9aaf39b25f9440a9aa9c5e5ada4dfe8 2024-11-27T21:13:52,150 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/A of cc764913a0f35f9f71e6ee97aeccf332 into d9aaf39b25f9440a9aa9c5e5ada4dfe8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:52,150 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:52,150 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/A, priority=13, startTime=1732742032021; duration=0sec 2024-11-27T21:13:52,150 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:13:52,150 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:A 2024-11-27T21:13:52,150 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:13:52,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/cb8fba5292f446a0bb3cbefe0f6f6818 is 50, key is test_row_0/B:col10/1732742032021/Put/seqid=0 2024-11-27T21:13:52,153 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:13:52,156 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/C is initiating minor compaction (all files) 2024-11-27T21:13:52,156 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/C in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:52,158 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/758588e2f15c445d87fee027a5d575c3, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/f56027a8cfa346dab16aef7f346a0c67, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/95cf6d43045643e180cc060b0be50db8] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=35.7 K 2024-11-27T21:13:52,159 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 758588e2f15c445d87fee027a5d575c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732742029738 2024-11-27T21:13:52,160 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting f56027a8cfa346dab16aef7f346a0c67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732742029882 2024-11-27T21:13:52,160 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 95cf6d43045643e180cc060b0be50db8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732742031169 2024-11-27T21:13:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T21:13:52,182 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#C#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:52,183 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/c1529fa421f94bc59dd2cdd0cf49644a is 50, key is test_row_0/C:col10/1732742031813/Put/seqid=0 2024-11-27T21:13:52,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741867_1043 (size=12151) 2024-11-27T21:13:52,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742092203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742092203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742092209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742092213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742092213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T21:13:52,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:52,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:52,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:52,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:52,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:52,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741868_1044 (size=12459) 2024-11-27T21:13:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T21:13:52,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T21:13:52,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:52,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:52,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:52,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:52,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742092419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742092420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742092420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742092429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742092429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,526 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/174ae1b327d84464adbb8254db9f5ffa as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/174ae1b327d84464adbb8254db9f5ffa 2024-11-27T21:13:52,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T21:13:52,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:52,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:52,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:52,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:52,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:52,540 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/B of cc764913a0f35f9f71e6ee97aeccf332 into 174ae1b327d84464adbb8254db9f5ffa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:52,540 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:52,540 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/B, priority=13, startTime=1732742032023; duration=0sec 2024-11-27T21:13:52,540 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:52,541 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:B 2024-11-27T21:13:52,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.69 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/cb8fba5292f446a0bb3cbefe0f6f6818 2024-11-27T21:13:52,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/635acaced14d4d619807664dab9ef09b is 50, key is test_row_0/C:col10/1732742032021/Put/seqid=0 2024-11-27T21:13:52,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741869_1045 (size=12151) 2024-11-27T21:13:52,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.69 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/635acaced14d4d619807664dab9ef09b 2024-11-27T21:13:52,643 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/c1529fa421f94bc59dd2cdd0cf49644a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/c1529fa421f94bc59dd2cdd0cf49644a 2024-11-27T21:13:52,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/398e722f967b4653a235f2e136dbf85f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/398e722f967b4653a235f2e136dbf85f 2024-11-27T21:13:52,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/398e722f967b4653a235f2e136dbf85f, entries=200, sequenceid=229, filesize=14.2 K 2024-11-27T21:13:52,658 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/C of cc764913a0f35f9f71e6ee97aeccf332 into c1529fa421f94bc59dd2cdd0cf49644a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:52,658 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:52,658 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/C, priority=13, startTime=1732742032024; duration=0sec 2024-11-27T21:13:52,658 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:52,658 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:C 2024-11-27T21:13:52,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/cb8fba5292f446a0bb3cbefe0f6f6818 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/cb8fba5292f446a0bb3cbefe0f6f6818 2024-11-27T21:13:52,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/cb8fba5292f446a0bb3cbefe0f6f6818, entries=150, sequenceid=229, filesize=11.9 K 2024-11-27T21:13:52,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/635acaced14d4d619807664dab9ef09b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/635acaced14d4d619807664dab9ef09b 2024-11-27T21:13:52,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/635acaced14d4d619807664dab9ef09b, entries=150, sequenceid=229, filesize=11.9 K 2024-11-27T21:13:52,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~137.05 KB/140344, heapSize ~385.91 KB/395168, currentSize=46.96 KB/48090 for cc764913a0f35f9f71e6ee97aeccf332 in 659ms, sequenceid=229, compaction requested=false 2024-11-27T21:13:52,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T21:13:52,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T21:13:52,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:52,692 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:13:52,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:52,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:52,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:52,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:52,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:52,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:52,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/fb17a9d4e96143a09b5ef914771c8c7a is 50, key is test_row_0/A:col10/1732742032077/Put/seqid=0 2024-11-27T21:13:52,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741870_1046 (size=12151) 2024-11-27T21:13:52,719 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/fb17a9d4e96143a09b5ef914771c8c7a 2024-11-27T21:13:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:52,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:52,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/02f7cbce7ffd4b34aa705edf103c8201 is 50, key is test_row_0/B:col10/1732742032077/Put/seqid=0 2024-11-27T21:13:52,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732742032759, max timestamp=1732742032775Num uniques 859; 2024-11-27T21:13:52,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732742032759, max timestamp=1732742032775Num uniques 859; 2024-11-27T21:13:52,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-27T21:13:52,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-27T21:13:52,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732742032759, max timestamp=1732742032775Num uniques 859; 2024-11-27T21:13:52,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-27T21:13:52,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741871_1047 (size=12151) 2024-11-27T21:13:52,801 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/02f7cbce7ffd4b34aa705edf103c8201 2024-11-27T21:13:52,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742092806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742092809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742092817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742092830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742092832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/b5225a7fb1f2474285dbbcdd12d917c4 is 50, key is test_row_0/C:col10/1732742032077/Put/seqid=0 2024-11-27T21:13:52,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741872_1048 (size=12151) 2024-11-27T21:13:52,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742092939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742092939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742092940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742092949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742092949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742093149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742093149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742093150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742093169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742093169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T21:13:53,272 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/b5225a7fb1f2474285dbbcdd12d917c4 2024-11-27T21:13:53,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/fb17a9d4e96143a09b5ef914771c8c7a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/fb17a9d4e96143a09b5ef914771c8c7a 2024-11-27T21:13:53,295 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/fb17a9d4e96143a09b5ef914771c8c7a, entries=150, sequenceid=242, filesize=11.9 K 2024-11-27T21:13:53,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/02f7cbce7ffd4b34aa705edf103c8201 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/02f7cbce7ffd4b34aa705edf103c8201 2024-11-27T21:13:53,306 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/02f7cbce7ffd4b34aa705edf103c8201, entries=150, sequenceid=242, filesize=11.9 K 2024-11-27T21:13:53,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/b5225a7fb1f2474285dbbcdd12d917c4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/b5225a7fb1f2474285dbbcdd12d917c4 2024-11-27T21:13:53,320 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/b5225a7fb1f2474285dbbcdd12d917c4, entries=150, sequenceid=242, filesize=11.9 K 2024-11-27T21:13:53,322 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for cc764913a0f35f9f71e6ee97aeccf332 in 631ms, sequenceid=242, compaction requested=true 2024-11-27T21:13:53,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:53,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:53,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-27T21:13:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-27T21:13:53,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-27T21:13:53,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2660 sec 2024-11-27T21:13:53,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.2780 sec 2024-11-27T21:13:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:53,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-27T21:13:53,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:53,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:53,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:53,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:53,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:53,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:53,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/5a622a6add11400a8d1a9a28f615b57e is 50, key is test_row_0/A:col10/1732742033470/Put/seqid=0 2024-11-27T21:13:53,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742093479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742093481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741873_1049 (size=12301) 2024-11-27T21:13:53,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/5a622a6add11400a8d1a9a28f615b57e 2024-11-27T21:13:53,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742093485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742093487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742093489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/7f0e1bbabb8949feae5aef29b96d0774 is 50, key is test_row_0/B:col10/1732742033470/Put/seqid=0 2024-11-27T21:13:53,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741874_1050 (size=12301) 2024-11-27T21:13:53,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/7f0e1bbabb8949feae5aef29b96d0774 2024-11-27T21:13:53,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/e896d628ca9a4b4ca07ed2b6d30bd01d is 50, key is test_row_0/C:col10/1732742033470/Put/seqid=0 2024-11-27T21:13:53,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741875_1051 (size=12301) 2024-11-27T21:13:53,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/e896d628ca9a4b4ca07ed2b6d30bd01d 2024-11-27T21:13:53,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/5a622a6add11400a8d1a9a28f615b57e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/5a622a6add11400a8d1a9a28f615b57e 2024-11-27T21:13:53,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/5a622a6add11400a8d1a9a28f615b57e, entries=150, sequenceid=277, filesize=12.0 K 2024-11-27T21:13:53,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/7f0e1bbabb8949feae5aef29b96d0774 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/7f0e1bbabb8949feae5aef29b96d0774 2024-11-27T21:13:53,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742093599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/7f0e1bbabb8949feae5aef29b96d0774, entries=150, sequenceid=277, filesize=12.0 K 2024-11-27T21:13:53,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742093600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742093600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742093600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/e896d628ca9a4b4ca07ed2b6d30bd01d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/e896d628ca9a4b4ca07ed2b6d30bd01d 2024-11-27T21:13:53,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/e896d628ca9a4b4ca07ed2b6d30bd01d, entries=150, sequenceid=277, filesize=12.0 K 2024-11-27T21:13:53,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for cc764913a0f35f9f71e6ee97aeccf332 in 149ms, sequenceid=277, compaction requested=true 2024-11-27T21:13:53,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:53,620 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:13:53,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:13:53,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:53,621 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:13:53,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:13:53,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:53,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:13:53,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:13:53,623 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51452 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:13:53,624 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/A is initiating minor compaction (all files) 2024-11-27T21:13:53,624 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/A in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:53,624 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/d9aaf39b25f9440a9aa9c5e5ada4dfe8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/398e722f967b4653a235f2e136dbf85f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/fb17a9d4e96143a09b5ef914771c8c7a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/5a622a6add11400a8d1a9a28f615b57e] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=50.2 K 2024-11-27T21:13:53,625 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:13:53,625 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/B is initiating minor compaction (all files) 2024-11-27T21:13:53,625 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/B in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:53,625 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/174ae1b327d84464adbb8254db9f5ffa, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/cb8fba5292f446a0bb3cbefe0f6f6818, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/02f7cbce7ffd4b34aa705edf103c8201, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/7f0e1bbabb8949feae5aef29b96d0774] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=47.9 K 2024-11-27T21:13:53,625 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9aaf39b25f9440a9aa9c5e5ada4dfe8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732742031169 2024-11-27T21:13:53,626 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 398e722f967b4653a235f2e136dbf85f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732742031963 2024-11-27T21:13:53,626 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 174ae1b327d84464adbb8254db9f5ffa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732742031169 2024-11-27T21:13:53,626 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb17a9d4e96143a09b5ef914771c8c7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732742032058 2024-11-27T21:13:53,627 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a622a6add11400a8d1a9a28f615b57e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732742032800 2024-11-27T21:13:53,627 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting cb8fba5292f446a0bb3cbefe0f6f6818, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732742031971 2024-11-27T21:13:53,628 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 02f7cbce7ffd4b34aa705edf103c8201, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732742032058 2024-11-27T21:13:53,631 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f0e1bbabb8949feae5aef29b96d0774, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732742032800 2024-11-27T21:13:53,664 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#A#compaction#39 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:53,665 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/e86a143f90be44dfabfb0729eac661f2 is 50, key is test_row_0/A:col10/1732742033470/Put/seqid=0 2024-11-27T21:13:53,670 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#B#compaction#40 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:53,671 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/ebee157f67ca4f0d8afa3ad4f6fc6fc4 is 50, key is test_row_0/B:col10/1732742033470/Put/seqid=0 2024-11-27T21:13:53,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741876_1052 (size=12745) 2024-11-27T21:13:53,720 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/e86a143f90be44dfabfb0729eac661f2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/e86a143f90be44dfabfb0729eac661f2 2024-11-27T21:13:53,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741877_1053 (size=12745) 2024-11-27T21:13:53,737 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/ebee157f67ca4f0d8afa3ad4f6fc6fc4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ebee157f67ca4f0d8afa3ad4f6fc6fc4 2024-11-27T21:13:53,740 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/A of cc764913a0f35f9f71e6ee97aeccf332 into e86a143f90be44dfabfb0729eac661f2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:53,740 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:53,740 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/A, priority=12, startTime=1732742033620; duration=0sec 2024-11-27T21:13:53,741 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:13:53,741 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:A 2024-11-27T21:13:53,741 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:13:53,743 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:13:53,743 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/C is initiating minor compaction (all files) 2024-11-27T21:13:53,744 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/C in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:53,744 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/c1529fa421f94bc59dd2cdd0cf49644a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/635acaced14d4d619807664dab9ef09b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/b5225a7fb1f2474285dbbcdd12d917c4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/e896d628ca9a4b4ca07ed2b6d30bd01d] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=47.9 K 2024-11-27T21:13:53,744 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1529fa421f94bc59dd2cdd0cf49644a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732742031169 2024-11-27T21:13:53,745 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 635acaced14d4d619807664dab9ef09b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732742031971 2024-11-27T21:13:53,745 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5225a7fb1f2474285dbbcdd12d917c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732742032058 2024-11-27T21:13:53,747 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/B of cc764913a0f35f9f71e6ee97aeccf332 into ebee157f67ca4f0d8afa3ad4f6fc6fc4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:53,747 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:53,747 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/B, priority=12, startTime=1732742033621; duration=0sec 2024-11-27T21:13:53,747 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:53,747 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:B 2024-11-27T21:13:53,749 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting e896d628ca9a4b4ca07ed2b6d30bd01d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732742032800 2024-11-27T21:13:53,772 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#C#compaction#41 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:53,773 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/d20c0722e7ca4de6807ad71bdd2a1436 is 50, key is test_row_0/C:col10/1732742033470/Put/seqid=0 2024-11-27T21:13:53,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741878_1054 (size=12745) 2024-11-27T21:13:53,798 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/d20c0722e7ca4de6807ad71bdd2a1436 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/d20c0722e7ca4de6807ad71bdd2a1436 2024-11-27T21:13:53,814 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/C of cc764913a0f35f9f71e6ee97aeccf332 into d20c0722e7ca4de6807ad71bdd2a1436(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:53,814 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:53,814 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/C, priority=12, startTime=1732742033622; duration=0sec 2024-11-27T21:13:53,815 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:53,815 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:C 2024-11-27T21:13:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:53,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:13:53,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:53,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:53,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:53,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:53,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:53,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:53,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/c7890064f19042bfa9a5657c8852703d is 50, key is test_row_0/A:col10/1732742033485/Put/seqid=0 2024-11-27T21:13:53,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741879_1055 (size=9857) 2024-11-27T21:13:53,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742093855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742093857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742093859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742093859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742093970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742093970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742093979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:53,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742093979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:54,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742093999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:54,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742094179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742094180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:54,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742094189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:54,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742094189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T21:13:54,219 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:13:54,220 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:13:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-27T21:13:54,224 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:13:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T21:13:54,225 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:13:54,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:13:54,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/c7890064f19042bfa9a5657c8852703d 2024-11-27T21:13:54,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/58a507b963034de99aa3d619b174b266 is 50, key is test_row_0/B:col10/1732742033485/Put/seqid=0 2024-11-27T21:13:54,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741880_1056 (size=9857) 2024-11-27T21:13:54,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/58a507b963034de99aa3d619b174b266 2024-11-27T21:13:54,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/853b1aaa959142a78c688f0f36f0474c is 50, key is test_row_0/C:col10/1732742033485/Put/seqid=0 2024-11-27T21:13:54,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741881_1057 (size=9857) 2024-11-27T21:13:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T21:13:54,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T21:13:54,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:54,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:54,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:54,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:54,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:54,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742094489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:54,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742094499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:54,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:54,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742094499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742094499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:54,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T21:13:54,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:54,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:54,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:54,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:54,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T21:13:54,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T21:13:54,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:54,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:54,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:54,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:54,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:54,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/853b1aaa959142a78c688f0f36f0474c 2024-11-27T21:13:54,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/c7890064f19042bfa9a5657c8852703d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c7890064f19042bfa9a5657c8852703d 2024-11-27T21:13:54,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c7890064f19042bfa9a5657c8852703d, entries=100, sequenceid=293, filesize=9.6 K 2024-11-27T21:13:54,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/58a507b963034de99aa3d619b174b266 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/58a507b963034de99aa3d619b174b266 2024-11-27T21:13:54,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/58a507b963034de99aa3d619b174b266, entries=100, sequenceid=293, filesize=9.6 K 2024-11-27T21:13:54,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/853b1aaa959142a78c688f0f36f0474c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/853b1aaa959142a78c688f0f36f0474c 2024-11-27T21:13:54,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/853b1aaa959142a78c688f0f36f0474c, entries=100, sequenceid=293, filesize=9.6 K 2024-11-27T21:13:54,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cc764913a0f35f9f71e6ee97aeccf332 in 938ms, sequenceid=293, compaction requested=false 2024-11-27T21:13:54,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:54,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T21:13:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:54,848 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T21:13:54,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:54,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:54,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:54,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:54,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:54,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T21:13:54,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/163f96db42274c9782196b955102eb90 is 50, key is test_row_0/A:col10/1732742033854/Put/seqid=0 2024-11-27T21:13:54,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741882_1058 (size=12301) 2024-11-27T21:13:54,879 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/163f96db42274c9782196b955102eb90 2024-11-27T21:13:54,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/c4547783ec43444c97521b0893ac4211 is 50, key is test_row_0/B:col10/1732742033854/Put/seqid=0 2024-11-27T21:13:54,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741883_1059 (size=12301) 2024-11-27T21:13:54,923 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/c4547783ec43444c97521b0893ac4211 2024-11-27T21:13:54,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/26a2672870324773bc7340013ece6881 is 50, key is test_row_0/C:col10/1732742033854/Put/seqid=0 2024-11-27T21:13:54,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741884_1060 (size=12301) 2024-11-27T21:13:54,966 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/26a2672870324773bc7340013ece6881 2024-11-27T21:13:54,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/163f96db42274c9782196b955102eb90 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/163f96db42274c9782196b955102eb90 2024-11-27T21:13:54,996 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/163f96db42274c9782196b955102eb90, entries=150, sequenceid=312, filesize=12.0 K 2024-11-27T21:13:54,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/c4547783ec43444c97521b0893ac4211 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/c4547783ec43444c97521b0893ac4211 2024-11-27T21:13:55,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:55,008 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/c4547783ec43444c97521b0893ac4211, entries=150, sequenceid=312, filesize=12.0 K 2024-11-27T21:13:55,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/26a2672870324773bc7340013ece6881 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/26a2672870324773bc7340013ece6881 2024-11-27T21:13:55,018 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/26a2672870324773bc7340013ece6881, entries=150, sequenceid=312, filesize=12.0 K 2024-11-27T21:13:55,020 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=20.13 KB/20610 for cc764913a0f35f9f71e6ee97aeccf332 in 172ms, sequenceid=312, compaction requested=true 2024-11-27T21:13:55,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:55,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:55,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-27T21:13:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-27T21:13:55,032 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-27T21:13:55,032 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 798 msec 2024-11-27T21:13:55,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:13:55,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:55,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:55,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:55,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:55,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:55,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:55,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 813 msec 2024-11-27T21:13:55,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/94709069c8a641b6b0e6d03bc54a7fa7 is 50, key is test_row_0/A:col10/1732742035021/Put/seqid=0 2024-11-27T21:13:55,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732742035052, max timestamp=1732742035078Num uniques 859; 2024-11-27T21:13:55,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-27T21:13:55,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742035058, max timestamp=1732742035081Num uniques 859; 2024-11-27T21:13:55,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:13:55,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742035058, max timestamp=1732742035081Num uniques 859; 2024-11-27T21:13:55,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:13:55,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741885_1061 (size=12301) 2024-11-27T21:13:55,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/94709069c8a641b6b0e6d03bc54a7fa7 2024-11-27T21:13:55,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/1d2ccea613574fb0a16779f89a1e0138 is 50, key is test_row_0/B:col10/1732742035021/Put/seqid=0 2024-11-27T21:13:55,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742095122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742095122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742095131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742095134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742095138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741886_1062 (size=12301) 2024-11-27T21:13:55,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742095250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742095250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742095250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742095250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742095259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T21:13:55,369 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:13:55,370 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:13:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-27T21:13:55,373 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:13:55,374 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:13:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T21:13:55,375 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:13:55,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742095459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742095459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742095460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742095460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T21:13:55,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742095479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T21:13:55,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:55,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:55,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:55,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:55,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:55,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/1d2ccea613574fb0a16779f89a1e0138 2024-11-27T21:13:55,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/7234985fc67f48c382bf8f5a86464e02 is 50, key is test_row_0/C:col10/1732742035021/Put/seqid=0 2024-11-27T21:13:55,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741887_1063 (size=12301) 2024-11-27T21:13:55,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/7234985fc67f48c382bf8f5a86464e02 2024-11-27T21:13:55,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/94709069c8a641b6b0e6d03bc54a7fa7 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/94709069c8a641b6b0e6d03bc54a7fa7 2024-11-27T21:13:55,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/94709069c8a641b6b0e6d03bc54a7fa7, entries=150, sequenceid=322, filesize=12.0 K 2024-11-27T21:13:55,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/1d2ccea613574fb0a16779f89a1e0138 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/1d2ccea613574fb0a16779f89a1e0138 2024-11-27T21:13:55,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/1d2ccea613574fb0a16779f89a1e0138, entries=150, sequenceid=322, filesize=12.0 K 2024-11-27T21:13:55,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/7234985fc67f48c382bf8f5a86464e02 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7234985fc67f48c382bf8f5a86464e02 2024-11-27T21:13:55,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7234985fc67f48c382bf8f5a86464e02, entries=150, sequenceid=322, filesize=12.0 K 2024-11-27T21:13:55,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=145.36 KB/148850 for cc764913a0f35f9f71e6ee97aeccf332 in 644ms, sequenceid=322, compaction requested=true 2024-11-27T21:13:55,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:55,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:13:55,678 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:13:55,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:55,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:13:55,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:55,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:13:55,678 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:13:55,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:13:55,680 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47204 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:13:55,680 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47204 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:13:55,680 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/B is initiating minor compaction (all files) 2024-11-27T21:13:55,680 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/A is initiating minor compaction (all files) 2024-11-27T21:13:55,680 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/B in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:55,680 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/A in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:55,680 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ebee157f67ca4f0d8afa3ad4f6fc6fc4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/58a507b963034de99aa3d619b174b266, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/c4547783ec43444c97521b0893ac4211, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/1d2ccea613574fb0a16779f89a1e0138] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=46.1 K 2024-11-27T21:13:55,680 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/e86a143f90be44dfabfb0729eac661f2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c7890064f19042bfa9a5657c8852703d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/163f96db42274c9782196b955102eb90, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/94709069c8a641b6b0e6d03bc54a7fa7] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=46.1 K 2024-11-27T21:13:55,681 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting ebee157f67ca4f0d8afa3ad4f6fc6fc4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732742032800 2024-11-27T21:13:55,681 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting e86a143f90be44dfabfb0729eac661f2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732742032800 2024-11-27T21:13:55,681 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 58a507b963034de99aa3d619b174b266, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732742033485 2024-11-27T21:13:55,682 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7890064f19042bfa9a5657c8852703d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732742033485 2024-11-27T21:13:55,682 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting c4547783ec43444c97521b0893ac4211, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732742033851 2024-11-27T21:13:55,682 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 163f96db42274c9782196b955102eb90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732742033851 2024-11-27T21:13:55,682 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d2ccea613574fb0a16779f89a1e0138, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732742035021 2024-11-27T21:13:55,683 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94709069c8a641b6b0e6d03bc54a7fa7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732742035021 2024-11-27T21:13:55,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T21:13:55,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:55,688 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=145.36 KB heapSize=410.38 KB 2024-11-27T21:13:55,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:55,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:55,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:55,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:55,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T21:13:55,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:55,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/43a5e2d869154fc39149f3f86684eb36 is 50, key is test_row_0/A:col10/1732742035117/Put/seqid=0 2024-11-27T21:13:55,707 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#A#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:55,708 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/112105a84f764f9ba2ef65bb07717675 is 50, key is test_row_0/A:col10/1732742035021/Put/seqid=0 2024-11-27T21:13:55,724 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#B#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:55,725 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/8e57d5bd961a43b5ad280cd6a9d9dc81 is 50, key is test_row_0/B:col10/1732742035021/Put/seqid=0 2024-11-27T21:13:55,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741888_1064 (size=12301) 2024-11-27T21:13:55,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741889_1065 (size=12881) 2024-11-27T21:13:55,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741890_1066 (size=12881) 2024-11-27T21:13:55,760 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/8e57d5bd961a43b5ad280cd6a9d9dc81 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/8e57d5bd961a43b5ad280cd6a9d9dc81 2024-11-27T21:13:55,771 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/B of cc764913a0f35f9f71e6ee97aeccf332 into 8e57d5bd961a43b5ad280cd6a9d9dc81(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:55,771 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:55,771 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/B, priority=12, startTime=1732742035678; duration=0sec 2024-11-27T21:13:55,771 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:13:55,771 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:B 2024-11-27T21:13:55,771 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:13:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:55,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:55,774 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47204 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:13:55,774 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/C is initiating minor compaction (all files) 2024-11-27T21:13:55,774 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/C in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:55,775 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/d20c0722e7ca4de6807ad71bdd2a1436, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/853b1aaa959142a78c688f0f36f0474c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/26a2672870324773bc7340013ece6881, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7234985fc67f48c382bf8f5a86464e02] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=46.1 K 2024-11-27T21:13:55,775 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting d20c0722e7ca4de6807ad71bdd2a1436, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732742032800 2024-11-27T21:13:55,776 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 853b1aaa959142a78c688f0f36f0474c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732742033485 2024-11-27T21:13:55,776 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 26a2672870324773bc7340013ece6881, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732742033851 2024-11-27T21:13:55,777 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 7234985fc67f48c382bf8f5a86464e02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732742035021 2024-11-27T21:13:55,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742095777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742095782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742095782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742095783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742095799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,803 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#C#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:55,804 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/029e5d8ac08d4bef965467d4a7e22cbb is 50, key is test_row_0/C:col10/1732742035021/Put/seqid=0 2024-11-27T21:13:55,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741891_1067 (size=12881) 2024-11-27T21:13:55,829 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/029e5d8ac08d4bef965467d4a7e22cbb as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/029e5d8ac08d4bef965467d4a7e22cbb 2024-11-27T21:13:55,843 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/C of cc764913a0f35f9f71e6ee97aeccf332 into 029e5d8ac08d4bef965467d4a7e22cbb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:55,843 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:55,843 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/C, priority=12, startTime=1732742035678; duration=0sec 2024-11-27T21:13:55,843 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:55,843 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:C 2024-11-27T21:13:55,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742095889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742095889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742095890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742095890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T21:13:56,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742096099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:56,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:56,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742096099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:56,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742096099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:56,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742096099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:56,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:56,129 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/43a5e2d869154fc39149f3f86684eb36 2024-11-27T21:13:56,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/4be89e570d58416c8b36037e66f52d4e is 50, key is test_row_0/B:col10/1732742035117/Put/seqid=0 2024-11-27T21:13:56,148 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/112105a84f764f9ba2ef65bb07717675 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/112105a84f764f9ba2ef65bb07717675 2024-11-27T21:13:56,156 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/A of cc764913a0f35f9f71e6ee97aeccf332 into 112105a84f764f9ba2ef65bb07717675(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:56,156 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:56,156 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/A, priority=12, startTime=1732742035677; duration=0sec 2024-11-27T21:13:56,157 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:56,157 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:A 2024-11-27T21:13:56,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741892_1068 (size=12301) 2024-11-27T21:13:56,160 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/4be89e570d58416c8b36037e66f52d4e 2024-11-27T21:13:56,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/28cb4a9b53314345824a5c38dce8b045 is 50, key is test_row_0/C:col10/1732742035117/Put/seqid=0 2024-11-27T21:13:56,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741893_1069 (size=12301) 2024-11-27T21:13:56,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742096319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:56,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:56,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742096409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:56,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742096409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:56,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742096419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:56,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742096420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T21:13:56,577 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/28cb4a9b53314345824a5c38dce8b045 2024-11-27T21:13:56,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/43a5e2d869154fc39149f3f86684eb36 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/43a5e2d869154fc39149f3f86684eb36 2024-11-27T21:13:56,595 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/43a5e2d869154fc39149f3f86684eb36, entries=150, sequenceid=359, filesize=12.0 K 2024-11-27T21:13:56,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/4be89e570d58416c8b36037e66f52d4e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/4be89e570d58416c8b36037e66f52d4e 2024-11-27T21:13:56,611 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/4be89e570d58416c8b36037e66f52d4e, entries=150, sequenceid=359, filesize=12.0 K 2024-11-27T21:13:56,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/28cb4a9b53314345824a5c38dce8b045 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/28cb4a9b53314345824a5c38dce8b045 2024-11-27T21:13:56,621 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/28cb4a9b53314345824a5c38dce8b045, entries=150, sequenceid=359, filesize=12.0 K 2024-11-27T21:13:56,623 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~410.33 KB/420176, currentSize=40.25 KB/41220 for cc764913a0f35f9f71e6ee97aeccf332 in 936ms, sequenceid=359, compaction requested=false 2024-11-27T21:13:56,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:56,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:56,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-27T21:13:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-27T21:13:56,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-27T21:13:56,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2520 sec 2024-11-27T21:13:56,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.2650 sec 2024-11-27T21:13:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:56,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:13:56,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:56,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:56,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:56,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:56,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:56,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:56,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/1e5f5713fb0a41fcaef7b9cd7a0b49f4 is 50, key is test_row_0/A:col10/1732742036930/Put/seqid=0 2024-11-27T21:13:56,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741894_1070 (size=14741) 2024-11-27T21:13:56,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/1e5f5713fb0a41fcaef7b9cd7a0b49f4 2024-11-27T21:13:56,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742036976, max timestamp=1732742036988Num uniques 859; 2024-11-27T21:13:56,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742036976, max timestamp=1732742036990Num uniques 859; 2024-11-27T21:13:56,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:13:56,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:13:56,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742036976, max timestamp=1732742036990Num uniques 859; 2024-11-27T21:13:56,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:13:57,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/2fa1df663b904bc6b04a7009143702f9 is 50, key is test_row_0/B:col10/1732742036930/Put/seqid=0 2024-11-27T21:13:57,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741895_1071 (size=12301) 2024-11-27T21:13:57,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742097045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742097048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732742037034, max timestamp=1732742037052Num uniques 1018; 2024-11-27T21:13:57,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-27T21:13:57,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732742037034, max timestamp=1732742037052Num uniques 1018; 2024-11-27T21:13:57,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-27T21:13:57,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732742037042, max timestamp=1732742037052Num uniques 1018; 2024-11-27T21:13:57,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-27T21:13:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742097144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742097152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742097169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742097179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742097259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742097269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742097339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,344 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in cc764913a0f35f9f71e6ee97aeccf332 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:13:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742097379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742097389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/2fa1df663b904bc6b04a7009143702f9 2024-11-27T21:13:57,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/6e6cc09ded124ec0b0344849fd0a9e13 is 50, key is test_row_0/C:col10/1732742036930/Put/seqid=0 2024-11-27T21:13:57,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741896_1072 (size=12301) 2024-11-27T21:13:57,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/6e6cc09ded124ec0b0344849fd0a9e13 2024-11-27T21:13:57,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/1e5f5713fb0a41fcaef7b9cd7a0b49f4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1e5f5713fb0a41fcaef7b9cd7a0b49f4 2024-11-27T21:13:57,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1e5f5713fb0a41fcaef7b9cd7a0b49f4, entries=200, sequenceid=372, filesize=14.4 K 2024-11-27T21:13:57,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/2fa1df663b904bc6b04a7009143702f9 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/2fa1df663b904bc6b04a7009143702f9 2024-11-27T21:13:57,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742097469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742097479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/2fa1df663b904bc6b04a7009143702f9, entries=150, sequenceid=372, filesize=12.0 K 2024-11-27T21:13:57,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/6e6cc09ded124ec0b0344849fd0a9e13 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/6e6cc09ded124ec0b0344849fd0a9e13 2024-11-27T21:13:57,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/6e6cc09ded124ec0b0344849fd0a9e13, entries=150, sequenceid=372, filesize=12.0 K 2024-11-27T21:13:57,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.14 KB/134292 for cc764913a0f35f9f71e6ee97aeccf332 in 567ms, sequenceid=372, compaction requested=true 2024-11-27T21:13:57,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:57,499 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:13:57,501 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:13:57,501 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/A is initiating minor compaction (all files) 2024-11-27T21:13:57,501 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/A in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:57,501 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/112105a84f764f9ba2ef65bb07717675, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/43a5e2d869154fc39149f3f86684eb36, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1e5f5713fb0a41fcaef7b9cd7a0b49f4] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=39.0 K 2024-11-27T21:13:57,502 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 112105a84f764f9ba2ef65bb07717675, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732742035021 2024-11-27T21:13:57,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:13:57,503 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43a5e2d869154fc39149f3f86684eb36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732742035117 2024-11-27T21:13:57,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:57,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:13:57,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:57,503 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:13:57,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:13:57,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:13:57,503 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e5f5713fb0a41fcaef7b9cd7a0b49f4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732742035776 2024-11-27T21:13:57,504 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:13:57,504 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/B is initiating minor compaction (all files) 2024-11-27T21:13:57,504 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/B in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:57,504 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/8e57d5bd961a43b5ad280cd6a9d9dc81, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/4be89e570d58416c8b36037e66f52d4e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/2fa1df663b904bc6b04a7009143702f9] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=36.6 K 2024-11-27T21:13:57,505 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e57d5bd961a43b5ad280cd6a9d9dc81, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732742035021 2024-11-27T21:13:57,505 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 4be89e570d58416c8b36037e66f52d4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732742035117 2024-11-27T21:13:57,506 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fa1df663b904bc6b04a7009143702f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732742035778 2024-11-27T21:13:57,520 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:57,521 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/88c90ce8d46d41d49ad0a2a28b70faf0 is 50, key is test_row_0/B:col10/1732742036930/Put/seqid=0 2024-11-27T21:13:57,522 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#A#compaction#61 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:57,523 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/f22f8155b4184c568994c65e17be86e7 is 50, key is test_row_0/A:col10/1732742036930/Put/seqid=0 2024-11-27T21:13:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T21:13:57,530 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:13:57,530 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:13:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-27T21:13:57,533 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:13:57,534 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:13:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T21:13:57,535 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:13:57,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741897_1073 (size=12983) 2024-11-27T21:13:57,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741898_1074 (size=12983) 2024-11-27T21:13:57,559 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/f22f8155b4184c568994c65e17be86e7 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/f22f8155b4184c568994c65e17be86e7 2024-11-27T21:13:57,568 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/A of cc764913a0f35f9f71e6ee97aeccf332 into f22f8155b4184c568994c65e17be86e7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:57,568 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:57,568 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/A, priority=13, startTime=1732742037499; duration=0sec 2024-11-27T21:13:57,568 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:13:57,568 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:A 2024-11-27T21:13:57,568 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:13:57,569 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:13:57,570 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/C is initiating minor compaction (all files) 2024-11-27T21:13:57,570 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/C in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:57,570 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/029e5d8ac08d4bef965467d4a7e22cbb, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/28cb4a9b53314345824a5c38dce8b045, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/6e6cc09ded124ec0b0344849fd0a9e13] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=36.6 K 2024-11-27T21:13:57,570 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 029e5d8ac08d4bef965467d4a7e22cbb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732742035021 2024-11-27T21:13:57,571 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28cb4a9b53314345824a5c38dce8b045, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732742035117 2024-11-27T21:13:57,571 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e6cc09ded124ec0b0344849fd0a9e13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732742035778 2024-11-27T21:13:57,579 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#C#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:13:57,579 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/2a3e415f3ca34ba89d687c5ef252f6cd is 50, key is test_row_0/C:col10/1732742036930/Put/seqid=0 2024-11-27T21:13:57,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741899_1075 (size=12983) 2024-11-27T21:13:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T21:13:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:57,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=137.85 KB heapSize=392.47 KB 2024-11-27T21:13:57,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:57,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:57,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:57,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:57,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:57,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:13:57,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T21:13:57,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:57,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:57,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:57,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:57,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:57,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/ce01229e49b14173960c48bf4706c5d7 is 50, key is test_row_0/A:col10/1732742037141/Put/seqid=0 2024-11-27T21:13:57,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741900_1076 (size=14741) 2024-11-27T21:13:57,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/ce01229e49b14173960c48bf4706c5d7 2024-11-27T21:13:57,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742097717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742097721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/83b9c5a3949245d4809990a5f3953e41 is 50, key is test_row_0/B:col10/1732742037141/Put/seqid=0 2024-11-27T21:13:57,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741901_1077 (size=12301) 2024-11-27T21:13:57,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742097789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742097789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742097829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742097829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:57,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:57,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T21:13:57,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T21:13:57,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:57,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:57,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:57,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:57,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:57,962 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/88c90ce8d46d41d49ad0a2a28b70faf0 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/88c90ce8d46d41d49ad0a2a28b70faf0 2024-11-27T21:13:57,973 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/B of cc764913a0f35f9f71e6ee97aeccf332 into 88c90ce8d46d41d49ad0a2a28b70faf0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:57,973 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:57,974 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/B, priority=13, startTime=1732742037503; duration=0sec 2024-11-27T21:13:57,974 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:57,974 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:B 2024-11-27T21:13:58,001 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/2a3e415f3ca34ba89d687c5ef252f6cd as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/2a3e415f3ca34ba89d687c5ef252f6cd 2024-11-27T21:13:58,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T21:13:58,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:58,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:58,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:58,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:58,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:58,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:58,009 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/C of cc764913a0f35f9f71e6ee97aeccf332 into 2a3e415f3ca34ba89d687c5ef252f6cd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:13:58,009 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:58,010 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/C, priority=13, startTime=1732742037503; duration=0sec 2024-11-27T21:13:58,010 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:13:58,010 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:C 2024-11-27T21:13:58,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:58,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742098039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:58,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742098039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:58,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:58,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:58,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:58,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/83b9c5a3949245d4809990a5f3953e41 2024-11-27T21:13:58,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/95e4a91b0c4d49a08bc7d2f74f2824dd is 50, key is test_row_0/C:col10/1732742037141/Put/seqid=0 2024-11-27T21:13:58,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T21:13:58,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:58,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:58,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:58,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:58,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T21:13:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:58,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741902_1078 (size=12301) 2024-11-27T21:13:58,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742098299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:58,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:58,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742098309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:58,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T21:13:58,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:58,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:58,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:58,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:58,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:58,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742098349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:58,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742098359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:58,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T21:13:58,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:58,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:58,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:58,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:58,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:58,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.93 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/95e4a91b0c4d49a08bc7d2f74f2824dd 2024-11-27T21:13:58,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/ce01229e49b14173960c48bf4706c5d7 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce01229e49b14173960c48bf4706c5d7 2024-11-27T21:13:58,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce01229e49b14173960c48bf4706c5d7, entries=200, sequenceid=427, filesize=14.4 K 2024-11-27T21:13:58,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/83b9c5a3949245d4809990a5f3953e41 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/83b9c5a3949245d4809990a5f3953e41 2024-11-27T21:13:58,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/83b9c5a3949245d4809990a5f3953e41, entries=150, sequenceid=427, filesize=12.0 K 2024-11-27T21:13:58,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/95e4a91b0c4d49a08bc7d2f74f2824dd as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/95e4a91b0c4d49a08bc7d2f74f2824dd 2024-11-27T21:13:58,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/95e4a91b0c4d49a08bc7d2f74f2824dd, entries=150, sequenceid=427, filesize=12.0 K 2024-11-27T21:13:58,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~137.85 KB/141162, heapSize ~392.42 KB/401840, currentSize=40.25 KB/41220 for cc764913a0f35f9f71e6ee97aeccf332 in 905ms, sequenceid=427, compaction requested=false 2024-11-27T21:13:58,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:58,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T21:13:58,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:58,620 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-27T21:13:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:13:58,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/0874b98afd92487eb25634f56206975b is 50, key is test_row_0/A:col10/1732742037704/Put/seqid=0 2024-11-27T21:13:58,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741903_1079 (size=12301) 2024-11-27T21:13:58,637 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/0874b98afd92487eb25634f56206975b 2024-11-27T21:13:58,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/22e185feead04c95842653e52409fb30 is 50, key is test_row_0/B:col10/1732742037704/Put/seqid=0 2024-11-27T21:13:58,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741904_1080 (size=12301) 2024-11-27T21:13:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T21:13:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:58,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:58,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742038898, max timestamp=1732742038910Num uniques 859; 2024-11-27T21:13:58,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:13:58,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742038898, max timestamp=1732742038910Num uniques 859; 2024-11-27T21:13:58,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:13:58,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742038898, max timestamp=1732742038910Num uniques 859; 2024-11-27T21:13:58,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:13:58,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:58,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742098988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:58,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:58,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:58,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:59,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742098996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:59,057 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/22e185feead04c95842653e52409fb30 2024-11-27T21:13:59,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/fa7c44b34bd648089debc2baae89bcb9 is 50, key is test_row_0/C:col10/1732742037704/Put/seqid=0 2024-11-27T21:13:59,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741905_1081 (size=12301) 2024-11-27T21:13:59,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:59,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742099099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:59,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742099109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:59,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742099309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:59,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:59,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:59,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742099319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742099319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:59,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742099319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:59,325 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_1' in cc764913a0f35f9f71e6ee97aeccf332 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:13:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:59,325 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_0' in cc764913a0f35f9f71e6ee97aeccf332 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:13:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:59,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742099358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:59,370 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in cc764913a0f35f9f71e6ee97aeccf332 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:13:59,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:59,475 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/fa7c44b34bd648089debc2baae89bcb9 2024-11-27T21:13:59,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/0874b98afd92487eb25634f56206975b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/0874b98afd92487eb25634f56206975b 2024-11-27T21:13:59,487 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/0874b98afd92487eb25634f56206975b, entries=150, sequenceid=438, filesize=12.0 K 2024-11-27T21:13:59,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/22e185feead04c95842653e52409fb30 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/22e185feead04c95842653e52409fb30 2024-11-27T21:13:59,494 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/22e185feead04c95842653e52409fb30, entries=150, sequenceid=438, filesize=12.0 K 2024-11-27T21:13:59,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/fa7c44b34bd648089debc2baae89bcb9 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/fa7c44b34bd648089debc2baae89bcb9 2024-11-27T21:13:59,501 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/fa7c44b34bd648089debc2baae89bcb9, entries=150, sequenceid=438, filesize=12.0 K 2024-11-27T21:13:59,502 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for cc764913a0f35f9f71e6ee97aeccf332 in 882ms, sequenceid=438, compaction requested=true 2024-11-27T21:13:59,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:13:59,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:59,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-27T21:13:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-27T21:13:59,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-27T21:13:59,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9690 sec 2024-11-27T21:13:59,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.9780 sec 2024-11-27T21:13:59,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:13:59,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-27T21:13:59,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:13:59,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:13:59,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:13:59,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:13:59,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:13:59,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:13:59,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/4cbc5191832b4d50ab519924eaa90490 is 50, key is test_row_0/A:col10/1732742039621/Put/seqid=0 2024-11-27T21:13:59,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741906_1082 (size=14741) 2024-11-27T21:13:59,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:59,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742099645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:59,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:59,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:59,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742099645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:59,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:59,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T21:13:59,689 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:13:59,689 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:13:59,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-27T21:13:59,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T21:13:59,692 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:13:59,692 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:13:59,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:13:59,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742099759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:59,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742099759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T21:13:59,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T21:13:59,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:59,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:13:59,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:13:59,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:59,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:13:59,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742099969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:59,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:13:59,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:13:59,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742099969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:13:59,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:13:59,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:00,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T21:14:00,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:00,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:00,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:00,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:00,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T21:14:00,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/4cbc5191832b4d50ab519924eaa90490 2024-11-27T21:14:00,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/742ecbe1b3654766abd6ea228d91b0f8 is 50, key is test_row_0/B:col10/1732742039621/Put/seqid=0 2024-11-27T21:14:00,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741907_1083 (size=12301) 2024-11-27T21:14:00,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/742ecbe1b3654766abd6ea228d91b0f8 2024-11-27T21:14:00,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/78f266dfd9c24fc9893294535904b878 is 50, key is test_row_0/C:col10/1732742039621/Put/seqid=0 2024-11-27T21:14:00,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741908_1084 (size=12301) 2024-11-27T21:14:00,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T21:14:00,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:00,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:00,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:00,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:00,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:00,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742100279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:00,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:00,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742100279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:00,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T21:14:00,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:00,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:00,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:00,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:00,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T21:14:00,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T21:14:00,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:00,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:00,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:00,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:00,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:00,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/78f266dfd9c24fc9893294535904b878 2024-11-27T21:14:00,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/4cbc5191832b4d50ab519924eaa90490 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/4cbc5191832b4d50ab519924eaa90490 2024-11-27T21:14:00,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/4cbc5191832b4d50ab519924eaa90490, entries=200, sequenceid=476, filesize=14.4 K 2024-11-27T21:14:00,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/742ecbe1b3654766abd6ea228d91b0f8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/742ecbe1b3654766abd6ea228d91b0f8 2024-11-27T21:14:00,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/742ecbe1b3654766abd6ea228d91b0f8, entries=150, sequenceid=476, filesize=12.0 K 2024-11-27T21:14:00,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/78f266dfd9c24fc9893294535904b878 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/78f266dfd9c24fc9893294535904b878 2024-11-27T21:14:00,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/78f266dfd9c24fc9893294535904b878, entries=150, sequenceid=476, filesize=12.0 K 2024-11-27T21:14:00,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for cc764913a0f35f9f71e6ee97aeccf332 in 900ms, sequenceid=476, compaction requested=true 2024-11-27T21:14:00,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:00,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:14:00,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:00,522 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:14:00,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:14:00,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:00,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:14:00,522 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:14:00,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:00,524 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:14:00,524 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54766 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:14:00,524 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/B is initiating minor compaction (all files) 2024-11-27T21:14:00,524 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/A is initiating minor compaction (all files) 2024-11-27T21:14:00,524 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/B in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:00,524 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/A in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:00,524 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/88c90ce8d46d41d49ad0a2a28b70faf0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/83b9c5a3949245d4809990a5f3953e41, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/22e185feead04c95842653e52409fb30, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/742ecbe1b3654766abd6ea228d91b0f8] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=48.7 K 2024-11-27T21:14:00,524 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/f22f8155b4184c568994c65e17be86e7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce01229e49b14173960c48bf4706c5d7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/0874b98afd92487eb25634f56206975b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/4cbc5191832b4d50ab519924eaa90490] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=53.5 K 2024-11-27T21:14:00,525 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 88c90ce8d46d41d49ad0a2a28b70faf0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732742035778 2024-11-27T21:14:00,525 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting f22f8155b4184c568994c65e17be86e7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732742035778 2024-11-27T21:14:00,525 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 83b9c5a3949245d4809990a5f3953e41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732742037141 2024-11-27T21:14:00,525 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce01229e49b14173960c48bf4706c5d7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732742037120 2024-11-27T21:14:00,526 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 22e185feead04c95842653e52409fb30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732742037704 2024-11-27T21:14:00,526 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0874b98afd92487eb25634f56206975b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732742037704 2024-11-27T21:14:00,526 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 742ecbe1b3654766abd6ea228d91b0f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732742038984 2024-11-27T21:14:00,527 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cbc5191832b4d50ab519924eaa90490, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732742038967 2024-11-27T21:14:00,547 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#A#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:00,547 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/e762cabe25da4d0883b554bdb9e6ebff is 50, key is test_row_0/A:col10/1732742039621/Put/seqid=0 2024-11-27T21:14:00,547 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#B#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:00,548 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/f2ff1816e4ca4150b3011a795e738357 is 50, key is test_row_0/B:col10/1732742039621/Put/seqid=0 2024-11-27T21:14:00,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741909_1085 (size=13119) 2024-11-27T21:14:00,569 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/e762cabe25da4d0883b554bdb9e6ebff as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/e762cabe25da4d0883b554bdb9e6ebff 2024-11-27T21:14:00,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741910_1086 (size=13119) 2024-11-27T21:14:00,577 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/A of cc764913a0f35f9f71e6ee97aeccf332 into e762cabe25da4d0883b554bdb9e6ebff(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:00,578 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:00,578 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/A, priority=12, startTime=1732742040522; duration=0sec 2024-11-27T21:14:00,578 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:00,578 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:A 2024-11-27T21:14:00,578 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:14:00,585 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/f2ff1816e4ca4150b3011a795e738357 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/f2ff1816e4ca4150b3011a795e738357 2024-11-27T21:14:00,585 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:14:00,585 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/C is initiating minor compaction (all files) 2024-11-27T21:14:00,585 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/C in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:00,585 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/2a3e415f3ca34ba89d687c5ef252f6cd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/95e4a91b0c4d49a08bc7d2f74f2824dd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/fa7c44b34bd648089debc2baae89bcb9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/78f266dfd9c24fc9893294535904b878] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=48.7 K 2024-11-27T21:14:00,586 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a3e415f3ca34ba89d687c5ef252f6cd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732742035778 2024-11-27T21:14:00,586 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95e4a91b0c4d49a08bc7d2f74f2824dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732742037141 2024-11-27T21:14:00,587 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa7c44b34bd648089debc2baae89bcb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732742037704 2024-11-27T21:14:00,588 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78f266dfd9c24fc9893294535904b878, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732742038984 2024-11-27T21:14:00,594 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/B of cc764913a0f35f9f71e6ee97aeccf332 into f2ff1816e4ca4150b3011a795e738357(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:00,594 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:00,594 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/B, priority=12, startTime=1732742040522; duration=0sec 2024-11-27T21:14:00,594 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:00,594 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:B 2024-11-27T21:14:00,602 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#C#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:00,603 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/54c6cd1a883649a2be244bdcb4df7277 is 50, key is test_row_0/C:col10/1732742039621/Put/seqid=0 2024-11-27T21:14:00,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T21:14:00,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:00,615 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-27T21:14:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:14:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:14:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:14:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:00,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/5547d871a1ca4eb296e9f7a5618f8629 is 50, key is test_row_0/A:col10/1732742039631/Put/seqid=0 2024-11-27T21:14:00,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741911_1087 (size=13119) 2024-11-27T21:14:00,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741912_1088 (size=9857) 2024-11-27T21:14:00,641 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/54c6cd1a883649a2be244bdcb4df7277 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/54c6cd1a883649a2be244bdcb4df7277 2024-11-27T21:14:00,650 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/C of cc764913a0f35f9f71e6ee97aeccf332 into 54c6cd1a883649a2be244bdcb4df7277(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:00,650 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:00,650 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/C, priority=12, startTime=1732742040522; duration=0sec 2024-11-27T21:14:00,651 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:00,651 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:C 2024-11-27T21:14:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:14:00,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T21:14:00,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742040824, max timestamp=1732742040843Num uniques 859; 2024-11-27T21:14:00,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742040824, max timestamp=1732742040843Num uniques 859; 2024-11-27T21:14:00,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:14:00,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:14:00,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742040824, max timestamp=1732742040843Num uniques 859; 2024-11-27T21:14:00,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:14:00,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742100887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:00,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742100888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:01,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742100999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:01,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:01,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742100999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:01,034 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/5547d871a1ca4eb296e9f7a5618f8629 2024-11-27T21:14:01,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/429a922e07f14b54accb16158c66d8f7 is 50, key is test_row_0/B:col10/1732742039631/Put/seqid=0 2024-11-27T21:14:01,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741913_1089 (size=9857) 2024-11-27T21:14:01,052 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/429a922e07f14b54accb16158c66d8f7 2024-11-27T21:14:01,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/b23ccedf453e4f59b182849916c8a8ae is 50, key is test_row_0/C:col10/1732742039631/Put/seqid=0 2024-11-27T21:14:01,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741914_1090 (size=9857) 2024-11-27T21:14:01,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742101209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:01,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:01,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742101209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:01,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:01,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:01,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742101349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:01,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742101349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:01,353 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_1' in cc764913a0f35f9f71e6ee97aeccf332 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:01,353 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_0' in cc764913a0f35f9f71e6ee97aeccf332 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:01,467 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/b23ccedf453e4f59b182849916c8a8ae 2024-11-27T21:14:01,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/5547d871a1ca4eb296e9f7a5618f8629 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/5547d871a1ca4eb296e9f7a5618f8629 2024-11-27T21:14:01,482 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/5547d871a1ca4eb296e9f7a5618f8629, entries=100, sequenceid=486, filesize=9.6 K 2024-11-27T21:14:01,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/429a922e07f14b54accb16158c66d8f7 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/429a922e07f14b54accb16158c66d8f7 2024-11-27T21:14:01,491 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/429a922e07f14b54accb16158c66d8f7, entries=100, sequenceid=486, filesize=9.6 K 2024-11-27T21:14:01,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/b23ccedf453e4f59b182849916c8a8ae as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/b23ccedf453e4f59b182849916c8a8ae 2024-11-27T21:14:01,498 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/b23ccedf453e4f59b182849916c8a8ae, entries=100, sequenceid=486, filesize=9.6 K 2024-11-27T21:14:01,499 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for cc764913a0f35f9f71e6ee97aeccf332 in 885ms, sequenceid=486, compaction requested=false 2024-11-27T21:14:01,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:01,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:01,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-27T21:14:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-27T21:14:01,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-27T21:14:01,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8070 sec 2024-11-27T21:14:01,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.8140 sec 2024-11-27T21:14:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:14:01,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=154.31 KB heapSize=436.08 KB 2024-11-27T21:14:01,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:14:01,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:14:01,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:14:01,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:14:01,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:14:01,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:14:01,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/beae42e66f424f5f9674b3dfb7b80fde is 50, key is test_row_0/A:col10/1732742040885/Put/seqid=0 2024-11-27T21:14:01,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742101536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:01,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741915_1091 (size=14741) 2024-11-27T21:14:01,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742101538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:01,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:01,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:01,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/beae42e66f424f5f9674b3dfb7b80fde 2024-11-27T21:14:01,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/6a23abd678d548c0b72297930b50621e is 50, key is test_row_0/B:col10/1732742040885/Put/seqid=0 2024-11-27T21:14:01,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741916_1092 (size=12301) 2024-11-27T21:14:01,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742101649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:01,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742101649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:01,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T21:14:01,839 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:01,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-27T21:14:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T21:14:01,843 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:01,844 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:01,844 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:01,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742101859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:01,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742101859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:01,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:01,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:01,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T21:14:01,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/6a23abd678d548c0b72297930b50621e 2024-11-27T21:14:01,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/ca864d3ea891497eb52e74853416c539 is 50, key is test_row_0/C:col10/1732742040885/Put/seqid=0 2024-11-27T21:14:01,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741917_1093 (size=12301) 2024-11-27T21:14:01,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T21:14:01,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:01,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:01,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:01,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:01,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:02,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T21:14:02,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:02,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:02,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:02,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:02,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T21:14:02,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742102169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:02,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742102169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:02,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T21:14:02,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:02,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:02,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:02,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:02,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:02,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/ca864d3ea891497eb52e74853416c539 2024-11-27T21:14:02,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/beae42e66f424f5f9674b3dfb7b80fde as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/beae42e66f424f5f9674b3dfb7b80fde 2024-11-27T21:14:02,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/beae42e66f424f5f9674b3dfb7b80fde, entries=200, sequenceid=525, filesize=14.4 K 2024-11-27T21:14:02,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/6a23abd678d548c0b72297930b50621e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6a23abd678d548c0b72297930b50621e 2024-11-27T21:14:02,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6a23abd678d548c0b72297930b50621e, entries=150, sequenceid=525, filesize=12.0 K 2024-11-27T21:14:02,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/ca864d3ea891497eb52e74853416c539 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/ca864d3ea891497eb52e74853416c539 2024-11-27T21:14:02,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/ca864d3ea891497eb52e74853416c539, entries=150, sequenceid=525, filesize=12.0 K 2024-11-27T21:14:02,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.03 KB/446496, currentSize=26.84 KB/27480 for cc764913a0f35f9f71e6ee97aeccf332 in 916ms, sequenceid=525, compaction requested=true 2024-11-27T21:14:02,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:02,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:14:02,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:02,442 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:02,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:14:02,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:02,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:14:02,442 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:02,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:02,445 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:02,445 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37717 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:02,445 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/B is initiating minor compaction (all files) 2024-11-27T21:14:02,445 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/A is initiating minor compaction (all files) 2024-11-27T21:14:02,445 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/B in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:02,445 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/A in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:02,445 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/f2ff1816e4ca4150b3011a795e738357, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/429a922e07f14b54accb16158c66d8f7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6a23abd678d548c0b72297930b50621e] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=34.5 K 2024-11-27T21:14:02,445 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/e762cabe25da4d0883b554bdb9e6ebff, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/5547d871a1ca4eb296e9f7a5618f8629, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/beae42e66f424f5f9674b3dfb7b80fde] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=36.8 K 2024-11-27T21:14:02,446 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting f2ff1816e4ca4150b3011a795e738357, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732742038984 2024-11-27T21:14:02,446 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting e762cabe25da4d0883b554bdb9e6ebff, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732742038984 2024-11-27T21:14:02,447 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 429a922e07f14b54accb16158c66d8f7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732742039631 2024-11-27T21:14:02,447 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5547d871a1ca4eb296e9f7a5618f8629, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732742039631 2024-11-27T21:14:02,447 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting beae42e66f424f5f9674b3dfb7b80fde, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1732742040877 2024-11-27T21:14:02,447 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a23abd678d548c0b72297930b50621e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1732742040877 2024-11-27T21:14:02,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T21:14:02,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:02,459 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-27T21:14:02,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:14:02,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:02,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:14:02,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:02,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:14:02,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:02,467 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#A#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:02,467 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/1589969d9ad9458ba11666f7478588ac is 50, key is test_row_0/A:col10/1732742040885/Put/seqid=0 2024-11-27T21:14:02,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T21:14:02,472 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#B#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:02,473 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/9b14062c6db44f918b1772faa259ed9a is 50, key is test_row_0/B:col10/1732742040885/Put/seqid=0 2024-11-27T21:14:02,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/fc9f61ef98fe4a8983a87e8c2cbd20f1 is 50, key is test_row_0/A:col10/1732742041535/Put/seqid=0 2024-11-27T21:14:02,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741918_1094 (size=13221) 2024-11-27T21:14:02,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741919_1095 (size=13221) 2024-11-27T21:14:02,528 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/9b14062c6db44f918b1772faa259ed9a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/9b14062c6db44f918b1772faa259ed9a 2024-11-27T21:14:02,533 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/1589969d9ad9458ba11666f7478588ac as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1589969d9ad9458ba11666f7478588ac 2024-11-27T21:14:02,536 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/B of cc764913a0f35f9f71e6ee97aeccf332 into 9b14062c6db44f918b1772faa259ed9a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:02,536 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:02,536 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/B, priority=13, startTime=1732742042442; duration=0sec 2024-11-27T21:14:02,537 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:02,537 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:B 2024-11-27T21:14:02,537 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:02,542 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:02,542 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/C is initiating minor compaction (all files) 2024-11-27T21:14:02,542 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/C in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:02,542 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/54c6cd1a883649a2be244bdcb4df7277, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/b23ccedf453e4f59b182849916c8a8ae, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/ca864d3ea891497eb52e74853416c539] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=34.5 K 2024-11-27T21:14:02,543 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 54c6cd1a883649a2be244bdcb4df7277, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732742038984 2024-11-27T21:14:02,543 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting b23ccedf453e4f59b182849916c8a8ae, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732742039631 2024-11-27T21:14:02,544 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting ca864d3ea891497eb52e74853416c539, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1732742040877 2024-11-27T21:14:02,547 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/A of cc764913a0f35f9f71e6ee97aeccf332 into 1589969d9ad9458ba11666f7478588ac(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:02,547 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:02,547 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/A, priority=13, startTime=1732742042442; duration=0sec 2024-11-27T21:14:02,547 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:02,548 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:A 2024-11-27T21:14:02,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741920_1096 (size=12301) 2024-11-27T21:14:02,556 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/fc9f61ef98fe4a8983a87e8c2cbd20f1 2024-11-27T21:14:02,557 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#C#compaction#84 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:02,559 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/46a1d22231e04f26926062788aeee460 is 50, key is test_row_0/C:col10/1732742040885/Put/seqid=0 2024-11-27T21:14:02,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/527f80d8e97a4248b5beb49877b48d31 is 50, key is test_row_0/B:col10/1732742041535/Put/seqid=0 2024-11-27T21:14:02,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741921_1097 (size=13221) 2024-11-27T21:14:02,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741922_1098 (size=12301) 2024-11-27T21:14:02,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:14:02,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742042714, max timestamp=1732742042733Num uniques 859; 2024-11-27T21:14:02,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:14:02,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742042714, max timestamp=1732742042733Num uniques 859; 2024-11-27T21:14:02,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742042714, max timestamp=1732742042733Num uniques 859; 2024-11-27T21:14:02,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:14:02,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:14:02,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732742042764, max timestamp=1732742042806Num uniques 1118; 2024-11-27T21:14:02,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-27T21:14:02,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732742042764, max timestamp=1732742042806Num uniques 1118; 2024-11-27T21:14:02,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-27T21:14:02,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732742042764, max timestamp=1732742042806Num uniques 1118; 2024-11-27T21:14:02,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-27T21:14:02,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732742042876, max timestamp=1732742042897Num uniques 1077; 2024-11-27T21:14:02,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-27T21:14:02,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732742042876, max timestamp=1732742042897Num uniques 1077; 2024-11-27T21:14:02,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-27T21:14:02,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732742042876, max timestamp=1732742042897Num uniques 1077; 2024-11-27T21:14:02,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-27T21:14:02,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732742042942, max timestamp=1732742042965Num uniques 1036; 2024-11-27T21:14:02,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-27T21:14:02,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732742042942, max timestamp=1732742042965Num uniques 1036; 2024-11-27T21:14:02,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732742042942, max timestamp=1732742042965Num uniques 1036; 2024-11-27T21:14:02,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-27T21:14:02,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-27T21:14:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T21:14:03,018 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/46a1d22231e04f26926062788aeee460 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/46a1d22231e04f26926062788aeee460 2024-11-27T21:14:03,022 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/527f80d8e97a4248b5beb49877b48d31 2024-11-27T21:14:03,028 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/C of cc764913a0f35f9f71e6ee97aeccf332 into 46a1d22231e04f26926062788aeee460(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:03,028 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:03,028 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/C, priority=13, startTime=1732742042442; duration=0sec 2024-11-27T21:14:03,029 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:03,029 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:C 2024-11-27T21:14:03,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732742042984, max timestamp=1732742043033Num uniques 1145; 2024-11-27T21:14:03,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-27T21:14:03,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732742042984, max timestamp=1732742043033Num uniques 1145; 2024-11-27T21:14:03,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-27T21:14:03,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732742042984, max timestamp=1732742043033Num uniques 1145; 2024-11-27T21:14:03,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-27T21:14:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/04e675b3aaa149d2a634d13509b800d4 is 50, key is test_row_0/C:col10/1732742041535/Put/seqid=0 2024-11-27T21:14:03,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741923_1099 (size=12301) 2024-11-27T21:14:03,067 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/04e675b3aaa149d2a634d13509b800d4 2024-11-27T21:14:03,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/fc9f61ef98fe4a8983a87e8c2cbd20f1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/fc9f61ef98fe4a8983a87e8c2cbd20f1 2024-11-27T21:14:03,085 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/fc9f61ef98fe4a8983a87e8c2cbd20f1, entries=150, sequenceid=532, filesize=12.0 K 2024-11-27T21:14:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/527f80d8e97a4248b5beb49877b48d31 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/527f80d8e97a4248b5beb49877b48d31 2024-11-27T21:14:03,097 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/527f80d8e97a4248b5beb49877b48d31, entries=150, sequenceid=532, filesize=12.0 K 2024-11-27T21:14:03,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/04e675b3aaa149d2a634d13509b800d4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/04e675b3aaa149d2a634d13509b800d4 2024-11-27T21:14:03,106 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/04e675b3aaa149d2a634d13509b800d4, entries=150, sequenceid=532, filesize=12.0 K 2024-11-27T21:14:03,108 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=134.18 KB/137400 for cc764913a0f35f9f71e6ee97aeccf332 in 649ms, sequenceid=532, compaction requested=false 2024-11-27T21:14:03,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:03,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:03,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-27T21:14:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:14:03,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=140.89 KB heapSize=396.84 KB 2024-11-27T21:14:03,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:14:03,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-27T21:14:03,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:14:03,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:03,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:14:03,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:03,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-27T21:14:03,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2660 sec 2024-11-27T21:14:03,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.2730 sec 2024-11-27T21:14:03,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/a831b082d8ae46848ac9f771e4fee1cc is 50, key is test_row_0/A:col10/1732742043103/Put/seqid=0 2024-11-27T21:14:03,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741924_1100 (size=17181) 2024-11-27T21:14:03,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742103147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:03,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742103154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:03,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742103259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:03,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742103269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:03,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:03,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52158 deadline: 1732742103389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:03,391 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in cc764913a0f35f9f71e6ee97aeccf332 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:03,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742103479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:03,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:03,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742103479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:03,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:03,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:03,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/a831b082d8ae46848ac9f771e4fee1cc 2024-11-27T21:14:03,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/910f67fc584c4e3b8234e117f5205afa is 50, key is test_row_0/B:col10/1732742043103/Put/seqid=0 2024-11-27T21:14:03,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741925_1101 (size=12301) 2024-11-27T21:14:03,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742103789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:03,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:03,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742103789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:03,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:03,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/910f67fc584c4e3b8234e117f5205afa 2024-11-27T21:14:03,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/869ceb7ca57a4edfa000d9fc19392f47 is 50, key is test_row_0/C:col10/1732742043103/Put/seqid=0 2024-11-27T21:14:03,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741926_1102 (size=12301) 2024-11-27T21:14:03,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T21:14:03,990 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:03,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-27T21:14:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T21:14:03,993 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:03,994 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:03,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T21:14:04,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T21:14:04,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:04,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:04,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:04,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:04,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:04,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:04,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T21:14:04,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:04,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:04,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:04,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:04,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:04,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742104299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:04,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742104299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:04,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T21:14:04,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/869ceb7ca57a4edfa000d9fc19392f47 2024-11-27T21:14:04,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/a831b082d8ae46848ac9f771e4fee1cc as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/a831b082d8ae46848ac9f771e4fee1cc 2024-11-27T21:14:04,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/a831b082d8ae46848ac9f771e4fee1cc, entries=250, sequenceid=640, filesize=16.8 K 2024-11-27T21:14:04,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/910f67fc584c4e3b8234e117f5205afa as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/910f67fc584c4e3b8234e117f5205afa 2024-11-27T21:14:04,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/910f67fc584c4e3b8234e117f5205afa, entries=150, sequenceid=640, filesize=12.0 K 2024-11-27T21:14:04,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/869ceb7ca57a4edfa000d9fc19392f47 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/869ceb7ca57a4edfa000d9fc19392f47 2024-11-27T21:14:04,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/869ceb7ca57a4edfa000d9fc19392f47, entries=150, sequenceid=640, filesize=12.0 K 2024-11-27T21:14:04,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~396.80 KB/406320, currentSize=40.25 KB/41220 for cc764913a0f35f9f71e6ee97aeccf332 in 1309ms, sequenceid=640, compaction requested=true 2024-11-27T21:14:04,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:04,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:14:04,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:04,418 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:04,418 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:04,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:14:04,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:04,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:14:04,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:04,419 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:04,419 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/A is initiating minor compaction (all files) 2024-11-27T21:14:04,419 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/A in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:04,420 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:04,420 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/B is initiating minor compaction (all files) 2024-11-27T21:14:04,420 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1589969d9ad9458ba11666f7478588ac, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/fc9f61ef98fe4a8983a87e8c2cbd20f1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/a831b082d8ae46848ac9f771e4fee1cc] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=41.7 K 2024-11-27T21:14:04,420 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/B in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:04,420 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/9b14062c6db44f918b1772faa259ed9a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/527f80d8e97a4248b5beb49877b48d31, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/910f67fc584c4e3b8234e117f5205afa] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=36.9 K 2024-11-27T21:14:04,422 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b14062c6db44f918b1772faa259ed9a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1732742040877 2024-11-27T21:14:04,422 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 527f80d8e97a4248b5beb49877b48d31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1732742041530 2024-11-27T21:14:04,422 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1589969d9ad9458ba11666f7478588ac, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1732742040877 2024-11-27T21:14:04,423 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc9f61ef98fe4a8983a87e8c2cbd20f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1732742041530 2024-11-27T21:14:04,423 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 910f67fc584c4e3b8234e117f5205afa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1732742043085 2024-11-27T21:14:04,423 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting a831b082d8ae46848ac9f771e4fee1cc, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1732742043071 2024-11-27T21:14:04,437 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#B#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:04,438 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/67d2fe55268a4e6d8e73bf76e5dd3701 is 50, key is test_row_0/B:col10/1732742043103/Put/seqid=0 2024-11-27T21:14:04,443 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#A#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:04,447 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/0fb8433f89a748068a555737a33e22db is 50, key is test_row_0/A:col10/1732742043103/Put/seqid=0 2024-11-27T21:14:04,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T21:14:04,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:04,458 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-27T21:14:04,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:14:04,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:04,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:14:04,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:04,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:14:04,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:04,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741927_1103 (size=13323) 2024-11-27T21:14:04,473 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/67d2fe55268a4e6d8e73bf76e5dd3701 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/67d2fe55268a4e6d8e73bf76e5dd3701 2024-11-27T21:14:04,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741928_1104 (size=13323) 2024-11-27T21:14:04,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/c6ad5dd4ffad4cef8f107e10aeccd9bd is 50, key is test_row_0/A:col10/1732742043152/Put/seqid=0 2024-11-27T21:14:04,484 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/0fb8433f89a748068a555737a33e22db as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/0fb8433f89a748068a555737a33e22db 2024-11-27T21:14:04,487 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/B of cc764913a0f35f9f71e6ee97aeccf332 into 67d2fe55268a4e6d8e73bf76e5dd3701(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:04,487 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:04,487 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/B, priority=13, startTime=1732742044418; duration=0sec 2024-11-27T21:14:04,488 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:04,488 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:B 2024-11-27T21:14:04,488 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:04,490 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:04,491 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/C is initiating minor compaction (all files) 2024-11-27T21:14:04,491 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/C in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:04,491 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/46a1d22231e04f26926062788aeee460, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/04e675b3aaa149d2a634d13509b800d4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/869ceb7ca57a4edfa000d9fc19392f47] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=36.9 K 2024-11-27T21:14:04,492 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/A of cc764913a0f35f9f71e6ee97aeccf332 into 0fb8433f89a748068a555737a33e22db(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:04,492 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:04,492 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/A, priority=13, startTime=1732742044417; duration=0sec 2024-11-27T21:14:04,492 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:04,492 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:A 2024-11-27T21:14:04,492 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 46a1d22231e04f26926062788aeee460, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1732742040877 2024-11-27T21:14:04,493 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 04e675b3aaa149d2a634d13509b800d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1732742041530 2024-11-27T21:14:04,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741929_1105 (size=9857) 2024-11-27T21:14:04,494 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 869ceb7ca57a4edfa000d9fc19392f47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1732742043085 2024-11-27T21:14:04,506 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#C#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:04,507 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/89e5ec1e7b6140008e9933a7338d8864 is 50, key is test_row_0/C:col10/1732742043103/Put/seqid=0 2024-11-27T21:14:04,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741930_1106 (size=13323) 2024-11-27T21:14:04,531 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/89e5ec1e7b6140008e9933a7338d8864 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/89e5ec1e7b6140008e9933a7338d8864 2024-11-27T21:14:04,541 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/C of cc764913a0f35f9f71e6ee97aeccf332 into 89e5ec1e7b6140008e9933a7338d8864(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:04,541 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:04,541 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/C, priority=13, startTime=1732742044418; duration=0sec 2024-11-27T21:14:04,541 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:04,541 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:C 2024-11-27T21:14:04,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T21:14:04,896 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/c6ad5dd4ffad4cef8f107e10aeccd9bd 2024-11-27T21:14:04,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/ee1e0f718c404bc5a4cfeb3a495bdfb0 is 50, key is test_row_0/B:col10/1732742043152/Put/seqid=0 2024-11-27T21:14:04,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741931_1107 (size=9857) 2024-11-27T21:14:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T21:14:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:14:05,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:05,336 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/ee1e0f718c404bc5a4cfeb3a495bdfb0 2024-11-27T21:14:05,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/647af83e392c4e77a113e5174aa0fab8 is 50, key is test_row_0/C:col10/1732742043152/Put/seqid=0 2024-11-27T21:14:05,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742045336, max timestamp=1732742045352Num uniques 859; 2024-11-27T21:14:05,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742045336, max timestamp=1732742045352Num uniques 859; 2024-11-27T21:14:05,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:14:05,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:14:05,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742045336, max timestamp=1732742045352Num uniques 859; 2024-11-27T21:14:05,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:14:05,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741932_1108 (size=9857) 2024-11-27T21:14:05,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742105392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:05,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742105393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:05,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742105394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:05,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742105400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:05,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742105509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:05,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742105509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:05,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:05,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742105509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:05,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:05,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:05,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742105510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:05,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742105720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:05,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:05,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742105720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:05,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:05,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:05,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742105720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:05,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:05,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742105720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:05,767 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/647af83e392c4e77a113e5174aa0fab8 2024-11-27T21:14:05,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/c6ad5dd4ffad4cef8f107e10aeccd9bd as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c6ad5dd4ffad4cef8f107e10aeccd9bd 2024-11-27T21:14:05,789 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c6ad5dd4ffad4cef8f107e10aeccd9bd, entries=100, sequenceid=649, filesize=9.6 K 2024-11-27T21:14:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/ee1e0f718c404bc5a4cfeb3a495bdfb0 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ee1e0f718c404bc5a4cfeb3a495bdfb0 2024-11-27T21:14:05,798 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ee1e0f718c404bc5a4cfeb3a495bdfb0, entries=100, sequenceid=649, filesize=9.6 K 2024-11-27T21:14:05,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/647af83e392c4e77a113e5174aa0fab8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/647af83e392c4e77a113e5174aa0fab8 2024-11-27T21:14:05,815 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/647af83e392c4e77a113e5174aa0fab8, entries=100, sequenceid=649, filesize=9.6 K 2024-11-27T21:14:05,817 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for cc764913a0f35f9f71e6ee97aeccf332 in 1358ms, sequenceid=649, compaction requested=false 2024-11-27T21:14:05,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:05,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:05,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-27T21:14:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-27T21:14:05,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-27T21:14:05,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8260 sec 2024-11-27T21:14:05,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.8340 sec 2024-11-27T21:14:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:14:06,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-27T21:14:06,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:14:06,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:06,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:14:06,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:06,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:14:06,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:06,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/cbd9bdb3caf741b38a0597fbc2214075 is 50, key is test_row_0/A:col10/1732742045392/Put/seqid=0 2024-11-27T21:14:06,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742106042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742106043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742106045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742106046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741933_1109 (size=14741) 2024-11-27T21:14:06,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/cbd9bdb3caf741b38a0597fbc2214075 2024-11-27T21:14:06,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/6ed6c97436f54459a8f38c2171a2c640 is 50, key is test_row_0/B:col10/1732742045392/Put/seqid=0 2024-11-27T21:14:06,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741934_1110 (size=12301) 2024-11-27T21:14:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T21:14:06,139 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:06,140 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-27T21:14:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T21:14:06,143 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:06,144 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:06,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:06,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742106159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742106159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742106159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T21:14:06,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T21:14:06,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:06,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:06,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:06,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:06,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:06,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742106379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742106379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742106379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T21:14:06,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:06,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:06,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:06,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:06,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T21:14:06,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/6ed6c97436f54459a8f38c2171a2c640 2024-11-27T21:14:06,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/8e3b321043fe46239aced0facead0926 is 50, key is test_row_0/C:col10/1732742045392/Put/seqid=0 2024-11-27T21:14:06,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741935_1111 (size=12301) 2024-11-27T21:14:06,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/8e3b321043fe46239aced0facead0926 2024-11-27T21:14:06,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/cbd9bdb3caf741b38a0597fbc2214075 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/cbd9bdb3caf741b38a0597fbc2214075 2024-11-27T21:14:06,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/cbd9bdb3caf741b38a0597fbc2214075, entries=200, sequenceid=689, filesize=14.4 K 2024-11-27T21:14:06,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/6ed6c97436f54459a8f38c2171a2c640 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6ed6c97436f54459a8f38c2171a2c640 2024-11-27T21:14:06,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6ed6c97436f54459a8f38c2171a2c640, entries=150, sequenceid=689, filesize=12.0 K 2024-11-27T21:14:06,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/8e3b321043fe46239aced0facead0926 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/8e3b321043fe46239aced0facead0926 2024-11-27T21:14:06,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/8e3b321043fe46239aced0facead0926, entries=150, sequenceid=689, filesize=12.0 K 2024-11-27T21:14:06,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for cc764913a0f35f9f71e6ee97aeccf332 in 519ms, sequenceid=689, compaction requested=true 2024-11-27T21:14:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:14:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:14:06,553 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc764913a0f35f9f71e6ee97aeccf332:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:14:06,553 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:06,557 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:06,557 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/B is initiating minor compaction (all files) 2024-11-27T21:14:06,558 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/B in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:06,558 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/67d2fe55268a4e6d8e73bf76e5dd3701, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ee1e0f718c404bc5a4cfeb3a495bdfb0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6ed6c97436f54459a8f38c2171a2c640] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=34.6 K 2024-11-27T21:14:06,558 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37921 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:06,558 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/A is initiating minor compaction (all files) 2024-11-27T21:14:06,558 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/A in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:06,558 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/0fb8433f89a748068a555737a33e22db, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c6ad5dd4ffad4cef8f107e10aeccd9bd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/cbd9bdb3caf741b38a0597fbc2214075] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=37.0 K 2024-11-27T21:14:06,558 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 67d2fe55268a4e6d8e73bf76e5dd3701, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1732742043085 2024-11-27T21:14:06,559 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting ee1e0f718c404bc5a4cfeb3a495bdfb0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1732742043145 2024-11-27T21:14:06,559 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fb8433f89a748068a555737a33e22db, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1732742043085 2024-11-27T21:14:06,560 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ed6c97436f54459a8f38c2171a2c640, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1732742045389 2024-11-27T21:14:06,561 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6ad5dd4ffad4cef8f107e10aeccd9bd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1732742043145 2024-11-27T21:14:06,561 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbd9bdb3caf741b38a0597fbc2214075, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1732742045389 2024-11-27T21:14:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:14:06,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:14:06,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:14:06,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:06,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:14:06,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:06,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:14:06,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:06,582 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:06,584 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/94edb8565cb7483b9a7745f2ca6cc59e is 50, key is test_row_0/A:col10/1732742045392/Put/seqid=0 2024-11-27T21:14:06,603 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:06,604 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/ce6e22ef639d4df490d25fbc76663681 is 50, key is test_row_0/B:col10/1732742045392/Put/seqid=0 2024-11-27T21:14:06,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T21:14:06,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:06,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:06,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:06,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:06,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:06,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/2250279e97d74c56a5e439021ee4f583 is 50, key is test_row_0/A:col10/1732742046568/Put/seqid=0 2024-11-27T21:14:06,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741936_1112 (size=13425) 2024-11-27T21:14:06,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741938_1114 (size=14741) 2024-11-27T21:14:06,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=699 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/2250279e97d74c56a5e439021ee4f583 2024-11-27T21:14:06,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741937_1113 (size=13425) 2024-11-27T21:14:06,654 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/ce6e22ef639d4df490d25fbc76663681 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ce6e22ef639d4df490d25fbc76663681 2024-11-27T21:14:06,656 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/94edb8565cb7483b9a7745f2ca6cc59e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/94edb8565cb7483b9a7745f2ca6cc59e 2024-11-27T21:14:06,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/e78b40b2a74344068a7334674aa68269 is 50, key is test_row_0/B:col10/1732742046568/Put/seqid=0 2024-11-27T21:14:06,665 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/B of cc764913a0f35f9f71e6ee97aeccf332 into ce6e22ef639d4df490d25fbc76663681(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:06,665 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:06,665 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/B, priority=13, startTime=1732742046553; duration=0sec 2024-11-27T21:14:06,665 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:06,665 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:B 2024-11-27T21:14:06,666 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:06,667 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:06,667 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cc764913a0f35f9f71e6ee97aeccf332/C is initiating minor compaction (all files) 2024-11-27T21:14:06,667 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc764913a0f35f9f71e6ee97aeccf332/C in TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:06,667 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/89e5ec1e7b6140008e9933a7338d8864, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/647af83e392c4e77a113e5174aa0fab8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/8e3b321043fe46239aced0facead0926] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp, totalSize=34.6 K 2024-11-27T21:14:06,669 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/A of cc764913a0f35f9f71e6ee97aeccf332 into 94edb8565cb7483b9a7745f2ca6cc59e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:06,669 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:06,669 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/A, priority=13, startTime=1732742046553; duration=0sec 2024-11-27T21:14:06,669 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:06,669 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:A 2024-11-27T21:14:06,669 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 89e5ec1e7b6140008e9933a7338d8864, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1732742043085 2024-11-27T21:14:06,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741939_1115 (size=12301) 2024-11-27T21:14:06,671 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 647af83e392c4e77a113e5174aa0fab8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1732742043145 2024-11-27T21:14:06,672 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e3b321043fe46239aced0facead0926, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1732742045389 2024-11-27T21:14:06,681 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc764913a0f35f9f71e6ee97aeccf332#C#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:06,682 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/46f30120bdbe4bd7b5038db56d1d5887 is 50, key is test_row_0/C:col10/1732742045392/Put/seqid=0 2024-11-27T21:14:06,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741940_1116 (size=13425) 2024-11-27T21:14:06,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742046676, max timestamp=1732742046696Num uniques 859; 2024-11-27T21:14:06,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:14:06,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742046676, max timestamp=1732742046697Num uniques 859; 2024-11-27T21:14:06,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:14:06,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742046676, max timestamp=1732742046697Num uniques 859; 2024-11-27T21:14:06,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:14:06,706 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/46f30120bdbe4bd7b5038db56d1d5887 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/46f30120bdbe4bd7b5038db56d1d5887 2024-11-27T21:14:06,716 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc764913a0f35f9f71e6ee97aeccf332/C of cc764913a0f35f9f71e6ee97aeccf332 into 46f30120bdbe4bd7b5038db56d1d5887(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:06,716 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:06,716 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., storeName=cc764913a0f35f9f71e6ee97aeccf332/C, priority=13, startTime=1732742046553; duration=0sec 2024-11-27T21:14:06,717 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:06,717 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc764913a0f35f9f71e6ee97aeccf332:C 2024-11-27T21:14:06,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742106738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742106739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742106741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742106742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T21:14:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:06,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T21:14:06,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742106849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742106859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742106859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742106860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:06,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T21:14:06,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:06,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:06,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:06,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:06,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:07,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742107059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:07,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:07,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742107069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742107069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:07,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T21:14:07,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:07,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:07,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:07,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:07,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:07,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:07,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:07,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:07,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742107069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:07,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:07,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:07,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:07,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:07,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:07,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:07,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=699 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/e78b40b2a74344068a7334674aa68269 2024-11-27T21:14:07,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/7134958f283d4ae39bb74108c351076b is 50, key is test_row_0/C:col10/1732742046568/Put/seqid=0 2024-11-27T21:14:07,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741941_1117 (size=12301) 2024-11-27T21:14:07,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T21:14:07,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:07,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:07,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:07,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:07,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T21:14:07,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1732742107369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:07,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T21:14:07,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:07,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:07,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:07,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:07,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:07,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52188 deadline: 1732742107379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:07,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52174 deadline: 1732742107379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:07,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52190 deadline: 1732742107380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:07,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=699 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/7134958f283d4ae39bb74108c351076b 2024-11-27T21:14:07,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/2250279e97d74c56a5e439021ee4f583 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/2250279e97d74c56a5e439021ee4f583 2024-11-27T21:14:07,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/2250279e97d74c56a5e439021ee4f583, entries=200, sequenceid=699, filesize=14.4 K 2024-11-27T21:14:07,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/e78b40b2a74344068a7334674aa68269 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/e78b40b2a74344068a7334674aa68269 2024-11-27T21:14:07,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/e78b40b2a74344068a7334674aa68269, entries=150, sequenceid=699, filesize=12.0 K 2024-11-27T21:14:07,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/7134958f283d4ae39bb74108c351076b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7134958f283d4ae39bb74108c351076b 2024-11-27T21:14:07,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7134958f283d4ae39bb74108c351076b, entries=150, sequenceid=699, filesize=12.0 K 2024-11-27T21:14:07,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cc764913a0f35f9f71e6ee97aeccf332 in 950ms, sequenceid=699, compaction requested=false 2024-11-27T21:14:07,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:07,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T21:14:07,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:07,530 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-27T21:14:07,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:14:07,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:07,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:14:07,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:07,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:14:07,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:07,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/5d4568ae49284b80bda42ca3da032bd8 is 50, key is test_row_0/A:col10/1732742046741/Put/seqid=0 2024-11-27T21:14:07,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741942_1118 (size=12301) 2024-11-27T21:14:07,752 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-27T21:14:07,752 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-27T21:14:07,752 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:07,752 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:07,753 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-27T21:14:07,753 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:07,756 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-27T21:14:07,756 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:07,757 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:07,757 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:07,757 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:07,757 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:07,758 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:07,758 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:07,758 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:07,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:07,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:07,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:07,759 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:07,759 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:07,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:14:07,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. as already flushing 2024-11-27T21:14:07,881 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-27T21:14:07,882 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:07,882 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:07,882 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:07,882 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:07,891 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-27T21:14:07,891 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-27T21:14:07,891 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:07,891 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:07,891 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:07,891 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:07,891 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:07,891 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:07,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:07,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:07,892 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-27T21:14:07,892 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:07,892 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:07,892 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:07,892 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:07,942 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=739 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/5d4568ae49284b80bda42ca3da032bd8 2024-11-27T21:14:07,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/1a17b206800a44c89e6f2ddeab43cd94 is 50, key is test_row_0/B:col10/1732742046741/Put/seqid=0 2024-11-27T21:14:07,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741943_1119 (size=12301) 2024-11-27T21:14:08,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T21:14:08,354 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=739 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/1a17b206800a44c89e6f2ddeab43cd94 2024-11-27T21:14:08,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/fd879d19d1cc4ec5bcd132b114624420 is 50, key is test_row_0/C:col10/1732742046741/Put/seqid=0 2024-11-27T21:14:08,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741944_1120 (size=12301) 2024-11-27T21:14:08,768 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=739 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/fd879d19d1cc4ec5bcd132b114624420 2024-11-27T21:14:08,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/5d4568ae49284b80bda42ca3da032bd8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/5d4568ae49284b80bda42ca3da032bd8 2024-11-27T21:14:08,778 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/5d4568ae49284b80bda42ca3da032bd8, entries=150, sequenceid=739, filesize=12.0 K 2024-11-27T21:14:08,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/1a17b206800a44c89e6f2ddeab43cd94 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/1a17b206800a44c89e6f2ddeab43cd94 2024-11-27T21:14:08,783 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/1a17b206800a44c89e6f2ddeab43cd94, entries=150, sequenceid=739, filesize=12.0 K 2024-11-27T21:14:08,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/fd879d19d1cc4ec5bcd132b114624420 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/fd879d19d1cc4ec5bcd132b114624420 2024-11-27T21:14:08,788 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/fd879d19d1cc4ec5bcd132b114624420, entries=150, sequenceid=739, filesize=12.0 K 2024-11-27T21:14:08,789 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=26.84 KB/27480 for cc764913a0f35f9f71e6ee97aeccf332 in 1258ms, sequenceid=739, compaction requested=true 2024-11-27T21:14:08,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for cc764913a0f35f9f71e6ee97aeccf332: 2024-11-27T21:14:08,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:08,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-27T21:14:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-27T21:14:08,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-27T21:14:08,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6460 sec 2024-11-27T21:14:08,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.6520 sec 2024-11-27T21:14:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T21:14:10,309 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:10,502 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T21:14:13,471 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-27T21:14:13,471 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:13,471 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:13,471 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:13,471 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:13,471 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T21:14:13,472 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-27T21:14:13,472 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-27T21:14:13,472 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-27T21:14:13,472 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 200 2024-11-27T21:14:13,472 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 198 2024-11-27T21:14:13,472 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T21:14:13,472 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2441 2024-11-27T21:14:13,472 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2539 2024-11-27T21:14:13,472 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T21:14:13,472 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1175 2024-11-27T21:14:13,472 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3514 rows 2024-11-27T21:14:13,472 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1163 2024-11-27T21:14:13,472 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3488 rows 2024-11-27T21:14:13,472 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T21:14:13,472 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:14:13,472 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:13,472 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:13,472 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:13,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T21:14:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T21:14:13,484 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742053483"}]},"ts":"1732742053483"} 2024-11-27T21:14:13,485 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T21:14:13,486 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T21:14:13,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:14:13,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc764913a0f35f9f71e6ee97aeccf332, UNASSIGN}] 2024-11-27T21:14:13,491 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc764913a0f35f9f71e6ee97aeccf332, UNASSIGN 2024-11-27T21:14:13,493 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=cc764913a0f35f9f71e6ee97aeccf332, regionState=CLOSING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:14:13,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc764913a0f35f9f71e6ee97aeccf332, UNASSIGN because future has completed 2024-11-27T21:14:13,495 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:14:13,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:14:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T21:14:13,651 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:14:13,651 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:14:13,652 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing cc764913a0f35f9f71e6ee97aeccf332, disabling compactions & flushes 2024-11-27T21:14:13,652 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:13,652 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:13,652 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. after waiting 0 ms 2024-11-27T21:14:13,652 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:13,652 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing cc764913a0f35f9f71e6ee97aeccf332 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-27T21:14:13,653 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=A 2024-11-27T21:14:13,653 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:13,653 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=B 2024-11-27T21:14:13,653 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:13,653 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc764913a0f35f9f71e6ee97aeccf332, store=C 2024-11-27T21:14:13,653 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:13,657 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/da57c888c0544d09959d8e17c1057d8a is 50, key is test_row_0/A:col10/1732742047889/Put/seqid=0 2024-11-27T21:14:13,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741945_1121 (size=12301) 2024-11-27T21:14:13,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T21:14:14,062 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/da57c888c0544d09959d8e17c1057d8a 2024-11-27T21:14:14,071 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/ad187fd8a3004b9eac5bdb0b02b63bea is 50, key is test_row_0/B:col10/1732742047889/Put/seqid=0 2024-11-27T21:14:14,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741946_1122 (size=12301) 2024-11-27T21:14:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T21:14:14,475 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/ad187fd8a3004b9eac5bdb0b02b63bea 2024-11-27T21:14:14,483 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/7b95ae40622e440e8c6d78928ef63a7e is 50, key is test_row_0/C:col10/1732742047889/Put/seqid=0 2024-11-27T21:14:14,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741947_1123 (size=12301) 2024-11-27T21:14:14,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T21:14:14,887 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/7b95ae40622e440e8c6d78928ef63a7e 2024-11-27T21:14:14,893 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/A/da57c888c0544d09959d8e17c1057d8a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/da57c888c0544d09959d8e17c1057d8a 2024-11-27T21:14:14,897 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/da57c888c0544d09959d8e17c1057d8a, entries=150, sequenceid=747, filesize=12.0 K 2024-11-27T21:14:14,898 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/B/ad187fd8a3004b9eac5bdb0b02b63bea as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ad187fd8a3004b9eac5bdb0b02b63bea 2024-11-27T21:14:14,902 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ad187fd8a3004b9eac5bdb0b02b63bea, entries=150, sequenceid=747, filesize=12.0 K 2024-11-27T21:14:14,903 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/.tmp/C/7b95ae40622e440e8c6d78928ef63a7e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7b95ae40622e440e8c6d78928ef63a7e 2024-11-27T21:14:14,907 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7b95ae40622e440e8c6d78928ef63a7e, entries=150, sequenceid=747, filesize=12.0 K 2024-11-27T21:14:14,908 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cc764913a0f35f9f71e6ee97aeccf332 in 1255ms, sequenceid=747, compaction requested=true 2024-11-27T21:14:14,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1c04f06cab2f46528d8846d928318fb4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce9216f0286d4cbc8361c7247b540557, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1c3c4aef223d4aba908295f573fded10, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce4974811b354f49a1b418b0b08a8c64, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/8035c3b5af6440cfb39d24189f8cdfe4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/34870c2ef1424b2a91e8283fb4a14f39, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/2242659bb7c24f24998025fa07f70520, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/997261afb33c42e6b11f8742f2c47a2e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c862038fa7ff49f6a670d2ac436f0447, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/d9aaf39b25f9440a9aa9c5e5ada4dfe8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/398e722f967b4653a235f2e136dbf85f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/fb17a9d4e96143a09b5ef914771c8c7a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/e86a143f90be44dfabfb0729eac661f2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/5a622a6add11400a8d1a9a28f615b57e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c7890064f19042bfa9a5657c8852703d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/163f96db42274c9782196b955102eb90, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/112105a84f764f9ba2ef65bb07717675, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/94709069c8a641b6b0e6d03bc54a7fa7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/43a5e2d869154fc39149f3f86684eb36, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1e5f5713fb0a41fcaef7b9cd7a0b49f4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/f22f8155b4184c568994c65e17be86e7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce01229e49b14173960c48bf4706c5d7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/0874b98afd92487eb25634f56206975b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/4cbc5191832b4d50ab519924eaa90490, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/e762cabe25da4d0883b554bdb9e6ebff, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/5547d871a1ca4eb296e9f7a5618f8629, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/beae42e66f424f5f9674b3dfb7b80fde, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1589969d9ad9458ba11666f7478588ac, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/fc9f61ef98fe4a8983a87e8c2cbd20f1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/a831b082d8ae46848ac9f771e4fee1cc, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/0fb8433f89a748068a555737a33e22db, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c6ad5dd4ffad4cef8f107e10aeccd9bd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/cbd9bdb3caf741b38a0597fbc2214075] to archive 2024-11-27T21:14:14,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:14:14,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1c04f06cab2f46528d8846d928318fb4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1c04f06cab2f46528d8846d928318fb4 2024-11-27T21:14:14,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce9216f0286d4cbc8361c7247b540557 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce9216f0286d4cbc8361c7247b540557 2024-11-27T21:14:14,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1c3c4aef223d4aba908295f573fded10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1c3c4aef223d4aba908295f573fded10 2024-11-27T21:14:14,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce4974811b354f49a1b418b0b08a8c64 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce4974811b354f49a1b418b0b08a8c64 2024-11-27T21:14:14,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/8035c3b5af6440cfb39d24189f8cdfe4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/8035c3b5af6440cfb39d24189f8cdfe4 2024-11-27T21:14:14,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/34870c2ef1424b2a91e8283fb4a14f39 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/34870c2ef1424b2a91e8283fb4a14f39 2024-11-27T21:14:14,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/2242659bb7c24f24998025fa07f70520 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/2242659bb7c24f24998025fa07f70520 2024-11-27T21:14:14,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/997261afb33c42e6b11f8742f2c47a2e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/997261afb33c42e6b11f8742f2c47a2e 2024-11-27T21:14:14,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c862038fa7ff49f6a670d2ac436f0447 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c862038fa7ff49f6a670d2ac436f0447 2024-11-27T21:14:14,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/d9aaf39b25f9440a9aa9c5e5ada4dfe8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/d9aaf39b25f9440a9aa9c5e5ada4dfe8 2024-11-27T21:14:14,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/398e722f967b4653a235f2e136dbf85f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/398e722f967b4653a235f2e136dbf85f 2024-11-27T21:14:14,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/fb17a9d4e96143a09b5ef914771c8c7a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/fb17a9d4e96143a09b5ef914771c8c7a 2024-11-27T21:14:14,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/e86a143f90be44dfabfb0729eac661f2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/e86a143f90be44dfabfb0729eac661f2 2024-11-27T21:14:14,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/5a622a6add11400a8d1a9a28f615b57e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/5a622a6add11400a8d1a9a28f615b57e 2024-11-27T21:14:14,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c7890064f19042bfa9a5657c8852703d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c7890064f19042bfa9a5657c8852703d 2024-11-27T21:14:14,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/163f96db42274c9782196b955102eb90 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/163f96db42274c9782196b955102eb90 2024-11-27T21:14:14,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/112105a84f764f9ba2ef65bb07717675 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/112105a84f764f9ba2ef65bb07717675 2024-11-27T21:14:14,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/94709069c8a641b6b0e6d03bc54a7fa7 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/94709069c8a641b6b0e6d03bc54a7fa7 2024-11-27T21:14:14,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/43a5e2d869154fc39149f3f86684eb36 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/43a5e2d869154fc39149f3f86684eb36 2024-11-27T21:14:14,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1e5f5713fb0a41fcaef7b9cd7a0b49f4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1e5f5713fb0a41fcaef7b9cd7a0b49f4 2024-11-27T21:14:14,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/f22f8155b4184c568994c65e17be86e7 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/f22f8155b4184c568994c65e17be86e7 2024-11-27T21:14:14,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce01229e49b14173960c48bf4706c5d7 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/ce01229e49b14173960c48bf4706c5d7 2024-11-27T21:14:14,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/0874b98afd92487eb25634f56206975b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/0874b98afd92487eb25634f56206975b 2024-11-27T21:14:14,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/4cbc5191832b4d50ab519924eaa90490 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/4cbc5191832b4d50ab519924eaa90490 2024-11-27T21:14:14,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/e762cabe25da4d0883b554bdb9e6ebff to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/e762cabe25da4d0883b554bdb9e6ebff 2024-11-27T21:14:14,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/5547d871a1ca4eb296e9f7a5618f8629 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/5547d871a1ca4eb296e9f7a5618f8629 2024-11-27T21:14:14,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/beae42e66f424f5f9674b3dfb7b80fde to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/beae42e66f424f5f9674b3dfb7b80fde 2024-11-27T21:14:14,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1589969d9ad9458ba11666f7478588ac to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/1589969d9ad9458ba11666f7478588ac 2024-11-27T21:14:14,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/fc9f61ef98fe4a8983a87e8c2cbd20f1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/fc9f61ef98fe4a8983a87e8c2cbd20f1 2024-11-27T21:14:14,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/a831b082d8ae46848ac9f771e4fee1cc to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/a831b082d8ae46848ac9f771e4fee1cc 2024-11-27T21:14:14,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/0fb8433f89a748068a555737a33e22db to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/0fb8433f89a748068a555737a33e22db 2024-11-27T21:14:14,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c6ad5dd4ffad4cef8f107e10aeccd9bd to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/c6ad5dd4ffad4cef8f107e10aeccd9bd 2024-11-27T21:14:14,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/cbd9bdb3caf741b38a0597fbc2214075 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/cbd9bdb3caf741b38a0597fbc2214075 2024-11-27T21:14:14,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/3b8bbb2b61224970886f6a851253dcc0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/4b312f79fffb439d905092483e7b12d9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/49e954ca358a4ced9c544666b45ba16d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6645439617a944ceac2d9d0ec9c785ab, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6f648534c10942fc939d1c97e27885fc, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/bed736206d0e404d968be73f2c1932a5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/5f11ead49d104f5fbfebc4f8621a04ad, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/71a726554bbe48bea347ccfe538bacac, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/174ae1b327d84464adbb8254db9f5ffa, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/85e174d7564b4885829ab4492d03d2b7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/cb8fba5292f446a0bb3cbefe0f6f6818, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/02f7cbce7ffd4b34aa705edf103c8201, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ebee157f67ca4f0d8afa3ad4f6fc6fc4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/7f0e1bbabb8949feae5aef29b96d0774, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/58a507b963034de99aa3d619b174b266, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/c4547783ec43444c97521b0893ac4211, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/8e57d5bd961a43b5ad280cd6a9d9dc81, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/1d2ccea613574fb0a16779f89a1e0138, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/4be89e570d58416c8b36037e66f52d4e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/88c90ce8d46d41d49ad0a2a28b70faf0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/2fa1df663b904bc6b04a7009143702f9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/83b9c5a3949245d4809990a5f3953e41, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/22e185feead04c95842653e52409fb30, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/f2ff1816e4ca4150b3011a795e738357, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/742ecbe1b3654766abd6ea228d91b0f8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/429a922e07f14b54accb16158c66d8f7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/9b14062c6db44f918b1772faa259ed9a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6a23abd678d548c0b72297930b50621e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/527f80d8e97a4248b5beb49877b48d31, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/67d2fe55268a4e6d8e73bf76e5dd3701, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/910f67fc584c4e3b8234e117f5205afa, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ee1e0f718c404bc5a4cfeb3a495bdfb0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6ed6c97436f54459a8f38c2171a2c640] to archive 2024-11-27T21:14:14,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:14:14,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/3b8bbb2b61224970886f6a851253dcc0 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/3b8bbb2b61224970886f6a851253dcc0 2024-11-27T21:14:14,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/4b312f79fffb439d905092483e7b12d9 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/4b312f79fffb439d905092483e7b12d9 2024-11-27T21:14:14,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/49e954ca358a4ced9c544666b45ba16d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/49e954ca358a4ced9c544666b45ba16d 2024-11-27T21:14:14,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6645439617a944ceac2d9d0ec9c785ab to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6645439617a944ceac2d9d0ec9c785ab 2024-11-27T21:14:14,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6f648534c10942fc939d1c97e27885fc to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6f648534c10942fc939d1c97e27885fc 2024-11-27T21:14:14,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/bed736206d0e404d968be73f2c1932a5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/bed736206d0e404d968be73f2c1932a5 2024-11-27T21:14:14,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/5f11ead49d104f5fbfebc4f8621a04ad to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/5f11ead49d104f5fbfebc4f8621a04ad 2024-11-27T21:14:14,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/71a726554bbe48bea347ccfe538bacac to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/71a726554bbe48bea347ccfe538bacac 2024-11-27T21:14:14,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/174ae1b327d84464adbb8254db9f5ffa to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/174ae1b327d84464adbb8254db9f5ffa 2024-11-27T21:14:14,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/85e174d7564b4885829ab4492d03d2b7 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/85e174d7564b4885829ab4492d03d2b7 2024-11-27T21:14:14,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/cb8fba5292f446a0bb3cbefe0f6f6818 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/cb8fba5292f446a0bb3cbefe0f6f6818 2024-11-27T21:14:14,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/02f7cbce7ffd4b34aa705edf103c8201 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/02f7cbce7ffd4b34aa705edf103c8201 2024-11-27T21:14:14,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ebee157f67ca4f0d8afa3ad4f6fc6fc4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ebee157f67ca4f0d8afa3ad4f6fc6fc4 2024-11-27T21:14:14,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/7f0e1bbabb8949feae5aef29b96d0774 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/7f0e1bbabb8949feae5aef29b96d0774 2024-11-27T21:14:14,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/58a507b963034de99aa3d619b174b266 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/58a507b963034de99aa3d619b174b266 2024-11-27T21:14:14,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/c4547783ec43444c97521b0893ac4211 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/c4547783ec43444c97521b0893ac4211 2024-11-27T21:14:14,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/8e57d5bd961a43b5ad280cd6a9d9dc81 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/8e57d5bd961a43b5ad280cd6a9d9dc81 2024-11-27T21:14:14,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/1d2ccea613574fb0a16779f89a1e0138 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/1d2ccea613574fb0a16779f89a1e0138 2024-11-27T21:14:14,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/4be89e570d58416c8b36037e66f52d4e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/4be89e570d58416c8b36037e66f52d4e 2024-11-27T21:14:14,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/88c90ce8d46d41d49ad0a2a28b70faf0 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/88c90ce8d46d41d49ad0a2a28b70faf0 2024-11-27T21:14:14,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/2fa1df663b904bc6b04a7009143702f9 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/2fa1df663b904bc6b04a7009143702f9 2024-11-27T21:14:15,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/83b9c5a3949245d4809990a5f3953e41 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/83b9c5a3949245d4809990a5f3953e41 2024-11-27T21:14:15,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/22e185feead04c95842653e52409fb30 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/22e185feead04c95842653e52409fb30 2024-11-27T21:14:15,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/f2ff1816e4ca4150b3011a795e738357 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/f2ff1816e4ca4150b3011a795e738357 2024-11-27T21:14:15,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/742ecbe1b3654766abd6ea228d91b0f8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/742ecbe1b3654766abd6ea228d91b0f8 2024-11-27T21:14:15,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/429a922e07f14b54accb16158c66d8f7 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/429a922e07f14b54accb16158c66d8f7 2024-11-27T21:14:15,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/9b14062c6db44f918b1772faa259ed9a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/9b14062c6db44f918b1772faa259ed9a 2024-11-27T21:14:15,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6a23abd678d548c0b72297930b50621e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6a23abd678d548c0b72297930b50621e 2024-11-27T21:14:15,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/527f80d8e97a4248b5beb49877b48d31 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/527f80d8e97a4248b5beb49877b48d31 2024-11-27T21:14:15,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/67d2fe55268a4e6d8e73bf76e5dd3701 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/67d2fe55268a4e6d8e73bf76e5dd3701 2024-11-27T21:14:15,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/910f67fc584c4e3b8234e117f5205afa to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/910f67fc584c4e3b8234e117f5205afa 2024-11-27T21:14:15,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ee1e0f718c404bc5a4cfeb3a495bdfb0 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ee1e0f718c404bc5a4cfeb3a495bdfb0 2024-11-27T21:14:15,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6ed6c97436f54459a8f38c2171a2c640 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/6ed6c97436f54459a8f38c2171a2c640 2024-11-27T21:14:15,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/21bba86033084555919a384735430167, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/60534b66d9464a33b4e9919226f7eca9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/de2b65f65b994843a4fa6501325c1adb, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/731990192ab84e3dbe89c6be62d151f3, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7bf0d1fb7cee4169a618d60d5faefc64, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/758588e2f15c445d87fee027a5d575c3, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/ca6c2fabebb945e0a2b5a4af130a9b1c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/f56027a8cfa346dab16aef7f346a0c67, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/c1529fa421f94bc59dd2cdd0cf49644a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/95cf6d43045643e180cc060b0be50db8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/635acaced14d4d619807664dab9ef09b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/b5225a7fb1f2474285dbbcdd12d917c4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/d20c0722e7ca4de6807ad71bdd2a1436, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/e896d628ca9a4b4ca07ed2b6d30bd01d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/853b1aaa959142a78c688f0f36f0474c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/26a2672870324773bc7340013ece6881, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/029e5d8ac08d4bef965467d4a7e22cbb, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7234985fc67f48c382bf8f5a86464e02, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/28cb4a9b53314345824a5c38dce8b045, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/2a3e415f3ca34ba89d687c5ef252f6cd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/6e6cc09ded124ec0b0344849fd0a9e13, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/95e4a91b0c4d49a08bc7d2f74f2824dd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/fa7c44b34bd648089debc2baae89bcb9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/54c6cd1a883649a2be244bdcb4df7277, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/78f266dfd9c24fc9893294535904b878, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/b23ccedf453e4f59b182849916c8a8ae, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/46a1d22231e04f26926062788aeee460, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/ca864d3ea891497eb52e74853416c539, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/04e675b3aaa149d2a634d13509b800d4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/89e5ec1e7b6140008e9933a7338d8864, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/869ceb7ca57a4edfa000d9fc19392f47, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/647af83e392c4e77a113e5174aa0fab8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/8e3b321043fe46239aced0facead0926] to archive 2024-11-27T21:14:15,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:14:15,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/21bba86033084555919a384735430167 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/21bba86033084555919a384735430167 2024-11-27T21:14:15,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/60534b66d9464a33b4e9919226f7eca9 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/60534b66d9464a33b4e9919226f7eca9 2024-11-27T21:14:15,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/de2b65f65b994843a4fa6501325c1adb to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/de2b65f65b994843a4fa6501325c1adb 2024-11-27T21:14:15,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/731990192ab84e3dbe89c6be62d151f3 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/731990192ab84e3dbe89c6be62d151f3 2024-11-27T21:14:15,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7bf0d1fb7cee4169a618d60d5faefc64 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7bf0d1fb7cee4169a618d60d5faefc64 2024-11-27T21:14:15,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/758588e2f15c445d87fee027a5d575c3 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/758588e2f15c445d87fee027a5d575c3 2024-11-27T21:14:15,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/ca6c2fabebb945e0a2b5a4af130a9b1c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/ca6c2fabebb945e0a2b5a4af130a9b1c 2024-11-27T21:14:15,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/f56027a8cfa346dab16aef7f346a0c67 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/f56027a8cfa346dab16aef7f346a0c67 2024-11-27T21:14:15,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/c1529fa421f94bc59dd2cdd0cf49644a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/c1529fa421f94bc59dd2cdd0cf49644a 2024-11-27T21:14:15,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/95cf6d43045643e180cc060b0be50db8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/95cf6d43045643e180cc060b0be50db8 2024-11-27T21:14:15,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/635acaced14d4d619807664dab9ef09b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/635acaced14d4d619807664dab9ef09b 2024-11-27T21:14:15,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/b5225a7fb1f2474285dbbcdd12d917c4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/b5225a7fb1f2474285dbbcdd12d917c4 2024-11-27T21:14:15,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/d20c0722e7ca4de6807ad71bdd2a1436 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/d20c0722e7ca4de6807ad71bdd2a1436 2024-11-27T21:14:15,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/e896d628ca9a4b4ca07ed2b6d30bd01d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/e896d628ca9a4b4ca07ed2b6d30bd01d 2024-11-27T21:14:15,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/853b1aaa959142a78c688f0f36f0474c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/853b1aaa959142a78c688f0f36f0474c 2024-11-27T21:14:15,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/26a2672870324773bc7340013ece6881 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/26a2672870324773bc7340013ece6881 2024-11-27T21:14:15,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/029e5d8ac08d4bef965467d4a7e22cbb to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/029e5d8ac08d4bef965467d4a7e22cbb 2024-11-27T21:14:15,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7234985fc67f48c382bf8f5a86464e02 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7234985fc67f48c382bf8f5a86464e02 2024-11-27T21:14:15,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/28cb4a9b53314345824a5c38dce8b045 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/28cb4a9b53314345824a5c38dce8b045 2024-11-27T21:14:15,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/2a3e415f3ca34ba89d687c5ef252f6cd to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/2a3e415f3ca34ba89d687c5ef252f6cd 2024-11-27T21:14:15,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/6e6cc09ded124ec0b0344849fd0a9e13 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/6e6cc09ded124ec0b0344849fd0a9e13 2024-11-27T21:14:15,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/95e4a91b0c4d49a08bc7d2f74f2824dd to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/95e4a91b0c4d49a08bc7d2f74f2824dd 2024-11-27T21:14:15,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/fa7c44b34bd648089debc2baae89bcb9 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/fa7c44b34bd648089debc2baae89bcb9 2024-11-27T21:14:15,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/54c6cd1a883649a2be244bdcb4df7277 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/54c6cd1a883649a2be244bdcb4df7277 2024-11-27T21:14:15,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/78f266dfd9c24fc9893294535904b878 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/78f266dfd9c24fc9893294535904b878 2024-11-27T21:14:15,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/b23ccedf453e4f59b182849916c8a8ae to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/b23ccedf453e4f59b182849916c8a8ae 2024-11-27T21:14:15,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/46a1d22231e04f26926062788aeee460 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/46a1d22231e04f26926062788aeee460 2024-11-27T21:14:15,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/ca864d3ea891497eb52e74853416c539 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/ca864d3ea891497eb52e74853416c539 2024-11-27T21:14:15,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/04e675b3aaa149d2a634d13509b800d4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/04e675b3aaa149d2a634d13509b800d4 2024-11-27T21:14:15,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/89e5ec1e7b6140008e9933a7338d8864 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/89e5ec1e7b6140008e9933a7338d8864 2024-11-27T21:14:15,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/869ceb7ca57a4edfa000d9fc19392f47 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/869ceb7ca57a4edfa000d9fc19392f47 2024-11-27T21:14:15,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/647af83e392c4e77a113e5174aa0fab8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/647af83e392c4e77a113e5174aa0fab8 2024-11-27T21:14:15,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/8e3b321043fe46239aced0facead0926 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/8e3b321043fe46239aced0facead0926 2024-11-27T21:14:15,060 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/recovered.edits/750.seqid, newMaxSeqId=750, maxSeqId=1 2024-11-27T21:14:15,061 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. 2024-11-27T21:14:15,061 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for cc764913a0f35f9f71e6ee97aeccf332: Waiting for close lock at 1732742053652Running coprocessor pre-close hooks at 1732742053652Disabling compacts and flushes for region at 1732742053652Disabling writes for close at 1732742053652Obtaining lock to block concurrent updates at 1732742053652Preparing flush snapshotting stores in cc764913a0f35f9f71e6ee97aeccf332 at 1732742053652Finished memstore snapshotting TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732742053653 (+1 ms)Flushing stores of TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332. at 1732742053654 (+1 ms)Flushing cc764913a0f35f9f71e6ee97aeccf332/A: creating writer at 1732742053654Flushing cc764913a0f35f9f71e6ee97aeccf332/A: appending metadata at 1732742053657 (+3 ms)Flushing cc764913a0f35f9f71e6ee97aeccf332/A: closing flushed file at 1732742053657Flushing cc764913a0f35f9f71e6ee97aeccf332/B: creating writer at 1732742054067 (+410 ms)Flushing cc764913a0f35f9f71e6ee97aeccf332/B: appending metadata at 1732742054070 (+3 ms)Flushing cc764913a0f35f9f71e6ee97aeccf332/B: closing flushed file at 1732742054070Flushing cc764913a0f35f9f71e6ee97aeccf332/C: creating writer at 1732742054480 (+410 ms)Flushing cc764913a0f35f9f71e6ee97aeccf332/C: appending metadata at 1732742054482 (+2 ms)Flushing cc764913a0f35f9f71e6ee97aeccf332/C: closing flushed file at 1732742054482Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e6d42b2: reopening flushed file at 1732742054892 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@693bfa72: reopening flushed file at 1732742054897 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f00987f: reopening flushed file at 1732742054902 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cc764913a0f35f9f71e6ee97aeccf332 in 1255ms, sequenceid=747, compaction requested=true at 1732742054908 (+6 ms)Writing region close event to WAL at 1732742055057 (+149 ms)Running coprocessor post-close hooks at 1732742055061 (+4 ms)Closed at 1732742055061 2024-11-27T21:14:15,063 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:14:15,064 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=cc764913a0f35f9f71e6ee97aeccf332, regionState=CLOSED 2024-11-27T21:14:15,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:14:15,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-27T21:14:15,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure cc764913a0f35f9f71e6ee97aeccf332, server=4585c39bed03,37381,1732742022265 in 1.5710 sec 2024-11-27T21:14:15,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-27T21:14:15,070 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc764913a0f35f9f71e6ee97aeccf332, UNASSIGN in 1.5770 sec 2024-11-27T21:14:15,072 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-27T21:14:15,072 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5830 sec 2024-11-27T21:14:15,073 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742055073"}]},"ts":"1732742055073"} 2024-11-27T21:14:15,074 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T21:14:15,074 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T21:14:15,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5970 sec 2024-11-27T21:14:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T21:14:15,629 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:15,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T21:14:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:15,637 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T21:14:15,639 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:15,644 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:14:15,648 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/recovered.edits] 2024-11-27T21:14:15,651 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/2250279e97d74c56a5e439021ee4f583 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/2250279e97d74c56a5e439021ee4f583 2024-11-27T21:14:15,653 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/5d4568ae49284b80bda42ca3da032bd8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/5d4568ae49284b80bda42ca3da032bd8 2024-11-27T21:14:15,654 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/94edb8565cb7483b9a7745f2ca6cc59e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/94edb8565cb7483b9a7745f2ca6cc59e 2024-11-27T21:14:15,655 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/da57c888c0544d09959d8e17c1057d8a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/A/da57c888c0544d09959d8e17c1057d8a 2024-11-27T21:14:15,658 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/1a17b206800a44c89e6f2ddeab43cd94 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/1a17b206800a44c89e6f2ddeab43cd94 2024-11-27T21:14:15,659 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ad187fd8a3004b9eac5bdb0b02b63bea to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ad187fd8a3004b9eac5bdb0b02b63bea 2024-11-27T21:14:15,660 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ce6e22ef639d4df490d25fbc76663681 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/ce6e22ef639d4df490d25fbc76663681 2024-11-27T21:14:15,662 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/e78b40b2a74344068a7334674aa68269 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/B/e78b40b2a74344068a7334674aa68269 2024-11-27T21:14:15,664 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/46f30120bdbe4bd7b5038db56d1d5887 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/46f30120bdbe4bd7b5038db56d1d5887 2024-11-27T21:14:15,665 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7134958f283d4ae39bb74108c351076b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7134958f283d4ae39bb74108c351076b 2024-11-27T21:14:15,667 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7b95ae40622e440e8c6d78928ef63a7e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/7b95ae40622e440e8c6d78928ef63a7e 2024-11-27T21:14:15,668 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/fd879d19d1cc4ec5bcd132b114624420 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/C/fd879d19d1cc4ec5bcd132b114624420 2024-11-27T21:14:15,671 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/recovered.edits/750.seqid to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332/recovered.edits/750.seqid 2024-11-27T21:14:15,672 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cc764913a0f35f9f71e6ee97aeccf332 2024-11-27T21:14:15,672 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T21:14:15,677 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:15,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-27T21:14:15,688 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T21:14:15,721 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T21:14:15,722 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:15,722 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T21:14:15,723 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732742055722"}]},"ts":"9223372036854775807"} 2024-11-27T21:14:15,726 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T21:14:15,726 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cc764913a0f35f9f71e6ee97aeccf332, NAME => 'TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T21:14:15,727 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T21:14:15,727 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732742055727"}]},"ts":"9223372036854775807"} 2024-11-27T21:14:15,730 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T21:14:15,731 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:15,732 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 98 msec 2024-11-27T21:14:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T21:14:15,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T21:14:15,750 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:15,763 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/4585c39bed03:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;4585c39bed03:37381-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1719228232_22 at /127.0.0.1:46214 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=380 (was 311) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9214 (was 9052) - AvailableMemoryMB LEAK? - 2024-11-27T21:14:15,774 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=380, ProcessCount=11, AvailableMemoryMB=9214 2024-11-27T21:14:15,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:14:15,776 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:14:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:15,779 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T21:14:15,779 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:15,779 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-27T21:14:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T21:14:15,780 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T21:14:15,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741948_1124 (size=1061) 2024-11-27T21:14:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T21:14:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T21:14:16,189 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de 2024-11-27T21:14:16,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741949_1125 (size=53) 2024-11-27T21:14:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T21:14:16,596 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:14:16,596 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6d7b62f727fc76cea63daea76cc211d5, disabling compactions & flushes 2024-11-27T21:14:16,596 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:16,596 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:16,596 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. after waiting 0 ms 2024-11-27T21:14:16,596 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:16,596 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:16,596 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6d7b62f727fc76cea63daea76cc211d5: Waiting for close lock at 1732742056596Disabling compacts and flushes for region at 1732742056596Disabling writes for close at 1732742056596Writing region close event to WAL at 1732742056596Closed at 1732742056596 2024-11-27T21:14:16,597 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T21:14:16,598 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732742056598"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732742056598"}]},"ts":"1732742056598"} 2024-11-27T21:14:16,599 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T21:14:16,600 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T21:14:16,600 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742056600"}]},"ts":"1732742056600"} 2024-11-27T21:14:16,602 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T21:14:16,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d7b62f727fc76cea63daea76cc211d5, ASSIGN}] 2024-11-27T21:14:16,603 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d7b62f727fc76cea63daea76cc211d5, ASSIGN 2024-11-27T21:14:16,604 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d7b62f727fc76cea63daea76cc211d5, ASSIGN; state=OFFLINE, location=4585c39bed03,37381,1732742022265; forceNewPlan=false, retain=false 2024-11-27T21:14:16,754 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=6d7b62f727fc76cea63daea76cc211d5, regionState=OPENING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:14:16,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d7b62f727fc76cea63daea76cc211d5, ASSIGN because future has completed 2024-11-27T21:14:16,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:14:16,911 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:16,912 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:14:16,912 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:16,912 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:14:16,912 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:16,912 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:16,914 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:16,915 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:14:16,915 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d7b62f727fc76cea63daea76cc211d5 columnFamilyName A 2024-11-27T21:14:16,916 DEBUG [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:16,916 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.HStore(327): Store=6d7b62f727fc76cea63daea76cc211d5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:14:16,916 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:16,917 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:14:16,917 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d7b62f727fc76cea63daea76cc211d5 columnFamilyName B 2024-11-27T21:14:16,917 DEBUG [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:16,918 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.HStore(327): Store=6d7b62f727fc76cea63daea76cc211d5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:14:16,918 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:16,919 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:14:16,919 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d7b62f727fc76cea63daea76cc211d5 columnFamilyName C 2024-11-27T21:14:16,919 DEBUG [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T21:14:16,919 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.HStore(327): Store=6d7b62f727fc76cea63daea76cc211d5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:14:16,919 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:16,920 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:16,920 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:16,920 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:16,921 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:16,921 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:16,922 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:14:16,923 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:16,925 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T21:14:16,926 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 6d7b62f727fc76cea63daea76cc211d5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58904726, jitterRate=-0.12225118279457092}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:14:16,926 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:16,926 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 6d7b62f727fc76cea63daea76cc211d5: Running coprocessor pre-open hook at 1732742056913Writing region info on filesystem at 1732742056913Initializing all the Stores at 1732742056913Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742056913Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742056913Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742056913Cleaning up temporary data from old regions at 1732742056921 (+8 ms)Running coprocessor post-open hooks at 1732742056926 (+5 ms)Region opened successfully at 1732742056926 2024-11-27T21:14:16,927 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., pid=34, masterSystemTime=1732742056908 2024-11-27T21:14:16,930 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=6d7b62f727fc76cea63daea76cc211d5, regionState=OPEN, openSeqNum=2, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:14:16,930 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:16,930 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:16,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:14:16,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-27T21:14:16,935 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 in 176 msec 2024-11-27T21:14:16,936 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-27T21:14:16,936 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d7b62f727fc76cea63daea76cc211d5, ASSIGN in 333 msec 2024-11-27T21:14:16,937 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T21:14:16,937 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742056937"}]},"ts":"1732742056937"} 2024-11-27T21:14:16,938 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T21:14:16,939 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T21:14:16,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-11-27T21:14:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T21:14:17,929 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:17,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:17,930 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:17,930 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:17,932 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:17,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:17,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:17,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e595508, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:17,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:17,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:17,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:17,934 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@329c9ead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:17,935 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:17,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:17,938 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T21:14:17,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 4585c39bed03,38503,1732742021440 2024-11-27T21:14:17,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1eb25f6a 2024-11-27T21:14:17,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T21:14:17,940 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T21:14:17,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:14:17,945 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:14:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:17,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741950_1126 (size=1097) 2024-11-27T21:14:18,363 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-27T21:14:18,363 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-27T21:14:18,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:14:18,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d7b62f727fc76cea63daea76cc211d5, REOPEN/MOVE}] 2024-11-27T21:14:18,374 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d7b62f727fc76cea63daea76cc211d5, REOPEN/MOVE 2024-11-27T21:14:18,374 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=6d7b62f727fc76cea63daea76cc211d5, regionState=CLOSING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:14:18,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d7b62f727fc76cea63daea76cc211d5, REOPEN/MOVE because future has completed 2024-11-27T21:14:18,376 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:14:18,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:14:18,529 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:18,529 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:14:18,529 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 6d7b62f727fc76cea63daea76cc211d5, disabling compactions & flushes 2024-11-27T21:14:18,529 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:18,529 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:18,529 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. after waiting 0 ms 2024-11-27T21:14:18,530 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:18,533 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T21:14:18,534 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:18,534 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 6d7b62f727fc76cea63daea76cc211d5: Waiting for close lock at 1732742058529Running coprocessor pre-close hooks at 1732742058529Disabling compacts and flushes for region at 1732742058529Disabling writes for close at 1732742058530 (+1 ms)Writing region close event to WAL at 1732742058530Running coprocessor post-close hooks at 1732742058534 (+4 ms)Closed at 1732742058534 2024-11-27T21:14:18,534 WARN [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 6d7b62f727fc76cea63daea76cc211d5 to self. 2024-11-27T21:14:18,536 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:18,537 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=6d7b62f727fc76cea63daea76cc211d5, regionState=CLOSED 2024-11-27T21:14:18,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:14:18,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-27T21:14:18,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 in 162 msec 2024-11-27T21:14:18,542 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d7b62f727fc76cea63daea76cc211d5, REOPEN/MOVE; state=CLOSED, location=4585c39bed03,37381,1732742022265; forceNewPlan=false, retain=true 2024-11-27T21:14:18,692 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=6d7b62f727fc76cea63daea76cc211d5, regionState=OPENING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:14:18,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d7b62f727fc76cea63daea76cc211d5, REOPEN/MOVE because future has completed 2024-11-27T21:14:18,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:14:18,850 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:18,850 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:14:18,850 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:18,850 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:14:18,850 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:18,850 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:18,852 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:18,853 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:14:18,857 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d7b62f727fc76cea63daea76cc211d5 columnFamilyName A 2024-11-27T21:14:18,859 DEBUG [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:18,859 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.HStore(327): Store=6d7b62f727fc76cea63daea76cc211d5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:14:18,860 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:18,861 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:14:18,861 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d7b62f727fc76cea63daea76cc211d5 columnFamilyName B 2024-11-27T21:14:18,861 DEBUG [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:18,861 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.HStore(327): Store=6d7b62f727fc76cea63daea76cc211d5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:14:18,862 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:18,862 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:14:18,862 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d7b62f727fc76cea63daea76cc211d5 columnFamilyName C 2024-11-27T21:14:18,862 DEBUG [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:18,863 INFO [StoreOpener-6d7b62f727fc76cea63daea76cc211d5-1 {}] regionserver.HStore(327): Store=6d7b62f727fc76cea63daea76cc211d5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:14:18,863 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:18,863 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:18,864 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:18,865 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:18,866 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:18,866 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:18,866 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:14:18,868 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:18,868 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 6d7b62f727fc76cea63daea76cc211d5; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75177288, jitterRate=0.12022888660430908}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:14:18,868 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:18,870 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 6d7b62f727fc76cea63daea76cc211d5: Running coprocessor pre-open hook at 1732742058850Writing region info on filesystem at 1732742058850Initializing all the Stores at 1732742058851 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742058851Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742058851Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742058851Cleaning up temporary data from old regions at 1732742058866 (+15 ms)Running coprocessor post-open hooks at 1732742058868 (+2 ms)Region opened successfully at 1732742058870 (+2 ms) 2024-11-27T21:14:18,871 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., pid=39, masterSystemTime=1732742058846 2024-11-27T21:14:18,873 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:18,873 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:18,873 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=6d7b62f727fc76cea63daea76cc211d5, regionState=OPEN, openSeqNum=5, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:14:18,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:14:18,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-27T21:14:18,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 in 182 msec 2024-11-27T21:14:18,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-27T21:14:18,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d7b62f727fc76cea63daea76cc211d5, REOPEN/MOVE in 505 msec 2024-11-27T21:14:18,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-27T21:14:18,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 515 msec 2024-11-27T21:14:18,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 938 msec 2024-11-27T21:14:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-27T21:14:18,888 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:18,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:18,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:18,890 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aaaa8e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:18,892 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:18,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@455bb67e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,893 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:18,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:18,894 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a6df162, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:18,895 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:18,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24dfa3ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:18,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:18,897 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3160ceb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:18,899 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771be477, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:18,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:18,901 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@585a36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:18,902 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:18,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19389d31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:18,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:18,904 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4988e638, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:18,905 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19aee7df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:18,906 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:18,907 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:18,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:18,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:18,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a6fd4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:18,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:18,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:18,908 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a68a25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:18,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:18,910 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@293659ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:18,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:18,911 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29db9dab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:18,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:18,913 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:18,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:18,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:18,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fe2886e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:18,914 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8d26930, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:18,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:18,917 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a8086e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:18,918 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:18,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d098d10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:18,919 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:18,919 DEBUG [Thread-642 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:18,919 DEBUG [Thread-645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:18,919 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:18,920 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:18,920 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:18,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:18,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:18,921 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:18,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:18,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:18,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-27T21:14:18,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:18,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:18,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:18,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:18,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T21:14:18,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:18,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:18,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:18,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:18,923 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:18,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:18,924 DEBUG [Thread-640 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:18,924 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:18,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:18,925 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:18,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:18,925 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:18,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:18,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:18,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:14:18,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:14:18,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:14:18,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:18,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:14:18,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:14:18,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:14:18,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:14:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:14:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:14:18,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:14:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:18,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:18,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:18,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:18,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:18,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:18,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:18,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127468fa2d086b94adeb2fd9ee680079314_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_1/A:col10/1732742058950/Put/seqid=0 2024-11-27T21:14:18,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741951_1127 (size=12154) 2024-11-27T21:14:18,999 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:19,008 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127468fa2d086b94adeb2fd9ee680079314_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127468fa2d086b94adeb2fd9ee680079314_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:19,009 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/d0ff7155b1254cae92d49aa385085d67, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:19,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/d0ff7155b1254cae92d49aa385085d67 is 175, key is test_row_1/A:col10/1732742058950/Put/seqid=0 2024-11-27T21:14:19,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741952_1128 (size=30951) 2024-11-27T21:14:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T21:14:19,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742119057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742119060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742119062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T21:14:19,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:19,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:19,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:19,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:19,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:19,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742119068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742119071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742119179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742119180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742119180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742119190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742119190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T21:14:19,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:19,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:19,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:19,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:19,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T21:14:19,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T21:14:19,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:19,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:19,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:19,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:19,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:19,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742119389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742119389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742119400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742119400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742119400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,428 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/d0ff7155b1254cae92d49aa385085d67 2024-11-27T21:14:19,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/af5c6a0ba8064ba1b0977899b53a160c is 50, key is test_row_1/B:col10/1732742058950/Put/seqid=0 2024-11-27T21:14:19,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741953_1129 (size=9657) 2024-11-27T21:14:19,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/af5c6a0ba8064ba1b0977899b53a160c 2024-11-27T21:14:19,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/c0fb4e691a2545b0af82875f400a38f5 is 50, key is test_row_1/C:col10/1732742058950/Put/seqid=0 2024-11-27T21:14:19,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741954_1130 (size=9657) 2024-11-27T21:14:19,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/c0fb4e691a2545b0af82875f400a38f5 2024-11-27T21:14:19,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/d0ff7155b1254cae92d49aa385085d67 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d0ff7155b1254cae92d49aa385085d67 2024-11-27T21:14:19,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T21:14:19,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:19,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:19,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:19,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:19,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d0ff7155b1254cae92d49aa385085d67, entries=150, sequenceid=16, filesize=30.2 K 2024-11-27T21:14:19,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:19,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/af5c6a0ba8064ba1b0977899b53a160c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/af5c6a0ba8064ba1b0977899b53a160c 2024-11-27T21:14:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T21:14:19,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/af5c6a0ba8064ba1b0977899b53a160c, entries=100, sequenceid=16, filesize=9.4 K 2024-11-27T21:14:19,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/c0fb4e691a2545b0af82875f400a38f5 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/c0fb4e691a2545b0af82875f400a38f5 2024-11-27T21:14:19,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/c0fb4e691a2545b0af82875f400a38f5, entries=100, sequenceid=16, filesize=9.4 K 2024-11-27T21:14:19,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6d7b62f727fc76cea63daea76cc211d5 in 620ms, sequenceid=16, compaction requested=false 2024-11-27T21:14:19,571 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T21:14:19,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:19,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T21:14:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:19,695 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-27T21:14:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:19,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a0443d3cc1ac4b38b710530d290a84ec_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742059059/Put/seqid=0 2024-11-27T21:14:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:19,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:19,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742119717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742119723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742119723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742119726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742119729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741955_1131 (size=12154) 2024-11-27T21:14:19,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742119828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742119839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742119839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742119839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:19,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742119839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:19,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:20,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742120039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:20,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742120049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:20,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742120050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:20,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:20,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742120050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:20,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742120050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:20,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:20,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:20,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T21:14:20,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:20,151 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a0443d3cc1ac4b38b710530d290a84ec_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a0443d3cc1ac4b38b710530d290a84ec_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:20,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/483a81bec8b54ad0bfb0302df6833262, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:20,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/483a81bec8b54ad0bfb0302df6833262 is 175, key is test_row_0/A:col10/1732742059059/Put/seqid=0 2024-11-27T21:14:20,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741956_1132 (size=30955) 2024-11-27T21:14:20,167 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/483a81bec8b54ad0bfb0302df6833262 2024-11-27T21:14:20,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/fcfa005c14e74c7981a1b712cd3fe4ea is 50, key is test_row_0/B:col10/1732742059059/Put/seqid=0 2024-11-27T21:14:20,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741957_1133 (size=12001) 2024-11-27T21:14:20,210 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/fcfa005c14e74c7981a1b712cd3fe4ea 2024-11-27T21:14:20,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/61a360ea542847d6a04c67a43130886f is 50, key is test_row_0/C:col10/1732742059059/Put/seqid=0 2024-11-27T21:14:20,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741958_1134 (size=12001) 2024-11-27T21:14:20,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742120349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:20,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:20,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742120359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742120360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:20,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742120360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:20,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:20,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742120360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:20,601 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T21:14:20,641 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/61a360ea542847d6a04c67a43130886f 2024-11-27T21:14:20,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/483a81bec8b54ad0bfb0302df6833262 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/483a81bec8b54ad0bfb0302df6833262 2024-11-27T21:14:20,657 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/483a81bec8b54ad0bfb0302df6833262, entries=150, sequenceid=36, filesize=30.2 K 2024-11-27T21:14:20,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/fcfa005c14e74c7981a1b712cd3fe4ea as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/fcfa005c14e74c7981a1b712cd3fe4ea 2024-11-27T21:14:20,664 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/fcfa005c14e74c7981a1b712cd3fe4ea, entries=150, sequenceid=36, filesize=11.7 K 2024-11-27T21:14:20,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/61a360ea542847d6a04c67a43130886f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/61a360ea542847d6a04c67a43130886f 2024-11-27T21:14:20,673 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/61a360ea542847d6a04c67a43130886f, entries=150, sequenceid=36, filesize=11.7 K 2024-11-27T21:14:20,674 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6d7b62f727fc76cea63daea76cc211d5 in 979ms, sequenceid=36, compaction requested=false 2024-11-27T21:14:20,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:20,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:20,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-27T21:14:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-27T21:14:20,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-27T21:14:20,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7520 sec 2024-11-27T21:14:20,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.7610 sec 2024-11-27T21:14:20,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-27T21:14:20,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:20,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:20,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:20,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:20,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:20,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:20,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127280b749e7b6b4d1faee1d26f2d022d13_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742060862/Put/seqid=0 2024-11-27T21:14:20,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741959_1135 (size=14594) 2024-11-27T21:14:20,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742120897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:20,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742120898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:20,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742120899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:20,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742120901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:20,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742120902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:21,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742121009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:21,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742121009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:21,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742121010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:21,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742121010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:21,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:21,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742121010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T21:14:21,069 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:21,069 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-27T21:14:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T21:14:21,071 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:21,072 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:21,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T21:14:21,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742121219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742121219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:21,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742121220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:21,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T21:14:21,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:21,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:21,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:21,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742121229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:21,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742121230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:21,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:21,296 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:21,300 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127280b749e7b6b4d1faee1d26f2d022d13_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127280b749e7b6b4d1faee1d26f2d022d13_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:21,301 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/4484eaad0c684544a6d7b3b51547bd4f, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:21,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/4484eaad0c684544a6d7b3b51547bd4f is 175, key is test_row_0/A:col10/1732742060862/Put/seqid=0 2024-11-27T21:14:21,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741960_1136 (size=39549) 2024-11-27T21:14:21,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T21:14:21,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:21,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:21,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:21,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T21:14:21,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742121528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:21,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742121529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:21,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742121529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:21,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T21:14:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:21,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742121539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:21,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742121539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:21,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:21,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:21,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:21,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T21:14:21,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:21,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:21,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:21,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T21:14:21,707 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/4484eaad0c684544a6d7b3b51547bd4f 2024-11-27T21:14:21,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/4b3bccaa4a9c4e64b749a9412162b025 is 50, key is test_row_0/B:col10/1732742060862/Put/seqid=0 2024-11-27T21:14:21,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741961_1137 (size=12001) 2024-11-27T21:14:21,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T21:14:21,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:21,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:21,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:21,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T21:14:21,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:21,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:21,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:21,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:22,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742122039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:22,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742122039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:22,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:22,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:22,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:22,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742122039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:22,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:22,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:22,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:22,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742122049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:22,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742122059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:22,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/4b3bccaa4a9c4e64b749a9412162b025 2024-11-27T21:14:22,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/41f6e58bbe384bf2a866ab14600c6702 is 50, key is test_row_0/C:col10/1732742060862/Put/seqid=0 2024-11-27T21:14:22,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741962_1138 (size=12001) 2024-11-27T21:14:22,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T21:14:22,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:22,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:22,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:22,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:22,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T21:14:22,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T21:14:22,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:22,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:22,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:22,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:22,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:22,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T21:14:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:22,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:22,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/41f6e58bbe384bf2a866ab14600c6702 2024-11-27T21:14:22,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/4484eaad0c684544a6d7b3b51547bd4f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/4484eaad0c684544a6d7b3b51547bd4f 2024-11-27T21:14:22,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/4484eaad0c684544a6d7b3b51547bd4f, entries=200, sequenceid=49, filesize=38.6 K 2024-11-27T21:14:22,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/4b3bccaa4a9c4e64b749a9412162b025 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/4b3bccaa4a9c4e64b749a9412162b025 2024-11-27T21:14:22,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/4b3bccaa4a9c4e64b749a9412162b025, entries=150, sequenceid=49, filesize=11.7 K 2024-11-27T21:14:22,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/41f6e58bbe384bf2a866ab14600c6702 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/41f6e58bbe384bf2a866ab14600c6702 2024-11-27T21:14:22,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/41f6e58bbe384bf2a866ab14600c6702, entries=150, sequenceid=49, filesize=11.7 K 2024-11-27T21:14:22,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 6d7b62f727fc76cea63daea76cc211d5 in 1695ms, sequenceid=49, compaction requested=true 2024-11-27T21:14:22,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:22,560 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:22,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:14:22,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:22,561 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:22,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:14:22,562 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:22,562 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/A is initiating minor compaction (all files) 2024-11-27T21:14:22,562 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/A in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:22,562 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d0ff7155b1254cae92d49aa385085d67, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/483a81bec8b54ad0bfb0302df6833262, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/4484eaad0c684544a6d7b3b51547bd4f] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=99.1 K 2024-11-27T21:14:22,562 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:22,562 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d0ff7155b1254cae92d49aa385085d67, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/483a81bec8b54ad0bfb0302df6833262, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/4484eaad0c684544a6d7b3b51547bd4f] 2024-11-27T21:14:22,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:22,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:14:22,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:22,563 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:22,563 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0ff7155b1254cae92d49aa385085d67, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732742058933 2024-11-27T21:14:22,563 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/B is initiating minor compaction (all files) 2024-11-27T21:14:22,563 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/B in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:22,563 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/af5c6a0ba8064ba1b0977899b53a160c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/fcfa005c14e74c7981a1b712cd3fe4ea, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/4b3bccaa4a9c4e64b749a9412162b025] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=32.9 K 2024-11-27T21:14:22,564 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 483a81bec8b54ad0bfb0302df6833262, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732742059022 2024-11-27T21:14:22,564 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting af5c6a0ba8064ba1b0977899b53a160c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732742058950 2024-11-27T21:14:22,565 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4484eaad0c684544a6d7b3b51547bd4f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732742059714 2024-11-27T21:14:22,565 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting fcfa005c14e74c7981a1b712cd3fe4ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732742059022 2024-11-27T21:14:22,566 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b3bccaa4a9c4e64b749a9412162b025, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732742059714 2024-11-27T21:14:22,577 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:22,582 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#B#compaction#121 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:22,583 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/a6d3fe13062146d1ac507224b8c9efb8 is 50, key is test_row_0/B:col10/1732742060862/Put/seqid=0 2024-11-27T21:14:22,584 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112785acf6560d3844338f022fb392ca39bb_6d7b62f727fc76cea63daea76cc211d5 store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:22,589 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112785acf6560d3844338f022fb392ca39bb_6d7b62f727fc76cea63daea76cc211d5, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:22,589 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112785acf6560d3844338f022fb392ca39bb_6d7b62f727fc76cea63daea76cc211d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:22,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T21:14:22,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:22,608 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-27T21:14:22,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:22,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:22,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:22,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:22,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:22,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:22,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127aedc98a53d1f43e494f6a36c7147f85c_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742060898/Put/seqid=0 2024-11-27T21:14:22,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741963_1139 (size=12104) 2024-11-27T21:14:22,638 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/a6d3fe13062146d1ac507224b8c9efb8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/a6d3fe13062146d1ac507224b8c9efb8 2024-11-27T21:14:22,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741964_1140 (size=4469) 2024-11-27T21:14:22,650 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/B of 6d7b62f727fc76cea63daea76cc211d5 into a6d3fe13062146d1ac507224b8c9efb8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:22,650 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:22,651 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/B, priority=13, startTime=1732742062561; duration=0sec 2024-11-27T21:14:22,651 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:22,651 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:B 2024-11-27T21:14:22,651 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:22,652 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:22,653 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/C is initiating minor compaction (all files) 2024-11-27T21:14:22,653 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/C in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:22,653 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/c0fb4e691a2545b0af82875f400a38f5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/61a360ea542847d6a04c67a43130886f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/41f6e58bbe384bf2a866ab14600c6702] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=32.9 K 2024-11-27T21:14:22,653 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting c0fb4e691a2545b0af82875f400a38f5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732742058950 2024-11-27T21:14:22,654 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 61a360ea542847d6a04c67a43130886f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732742059022 2024-11-27T21:14:22,654 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 41f6e58bbe384bf2a866ab14600c6702, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732742059714 2024-11-27T21:14:22,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741965_1141 (size=12154) 2024-11-27T21:14:22,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,674 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127aedc98a53d1f43e494f6a36c7147f85c_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127aedc98a53d1f43e494f6a36c7147f85c_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:22,675 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#C#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:22,675 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/0b88b1902a2248a79cefd87bd1514332 is 50, key is test_row_0/C:col10/1732742060862/Put/seqid=0 2024-11-27T21:14:22,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/eaa10a5ee58249439495be8b8f95bfa4, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:22,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/eaa10a5ee58249439495be8b8f95bfa4 is 175, key is test_row_0/A:col10/1732742060898/Put/seqid=0 2024-11-27T21:14:22,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741966_1142 (size=12104) 2024-11-27T21:14:22,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741967_1143 (size=30955) 2024-11-27T21:14:22,697 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/0b88b1902a2248a79cefd87bd1514332 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/0b88b1902a2248a79cefd87bd1514332 2024-11-27T21:14:22,698 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/eaa10a5ee58249439495be8b8f95bfa4 2024-11-27T21:14:22,709 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/C of 6d7b62f727fc76cea63daea76cc211d5 into 0b88b1902a2248a79cefd87bd1514332(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:22,709 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:22,709 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/C, priority=13, startTime=1732742062563; duration=0sec 2024-11-27T21:14:22,709 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:22,709 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:C 2024-11-27T21:14:22,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/9f3a2656bfd44eaf89967422c94343c8 is 50, key is test_row_0/B:col10/1732742060898/Put/seqid=0 2024-11-27T21:14:22,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741968_1144 (size=12001) 2024-11-27T21:14:22,717 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/9f3a2656bfd44eaf89967422c94343c8 2024-11-27T21:14:22,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/1d598808def341098605a26b746245b9 is 50, key is test_row_0/C:col10/1732742060898/Put/seqid=0 2024-11-27T21:14:22,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741969_1145 (size=12001) 2024-11-27T21:14:22,734 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/1d598808def341098605a26b746245b9 2024-11-27T21:14:22,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/eaa10a5ee58249439495be8b8f95bfa4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eaa10a5ee58249439495be8b8f95bfa4 2024-11-27T21:14:22,749 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eaa10a5ee58249439495be8b8f95bfa4, entries=150, sequenceid=69, filesize=30.2 K 2024-11-27T21:14:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/9f3a2656bfd44eaf89967422c94343c8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/9f3a2656bfd44eaf89967422c94343c8 2024-11-27T21:14:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,757 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/9f3a2656bfd44eaf89967422c94343c8, entries=150, sequenceid=69, filesize=11.7 K 2024-11-27T21:14:22,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/1d598808def341098605a26b746245b9 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/1d598808def341098605a26b746245b9 2024-11-27T21:14:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,768 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/1d598808def341098605a26b746245b9, entries=150, sequenceid=69, filesize=11.7 K 2024-11-27T21:14:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,769 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 6d7b62f727fc76cea63daea76cc211d5 in 161ms, sequenceid=69, compaction requested=false 2024-11-27T21:14:22,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:22,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:22,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-27T21:14:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-27T21:14:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-27T21:14:22,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6990 sec 2024-11-27T21:14:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.7050 sec 2024-11-27T21:14:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,047 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#A#compaction#120 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,048 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/f4ac37da4d8f4aab97b7ec8caf835cb1 is 175, key is test_row_0/A:col10/1732742060862/Put/seqid=0 2024-11-27T21:14:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741970_1146 (size=31058) 2024-11-27T21:14:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:14:23,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:23,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:23,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:23,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:23,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:23,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b97745bf5d9848b3874e20717cf9847a_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742063090/Put/seqid=0 2024-11-27T21:14:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741971_1147 (size=21918) 2024-11-27T21:14:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1732742063092, max timestamp=1732742063138Num uniques 859; 2024-11-27T21:14:23,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-27T21:14:23,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1732742063092, max timestamp=1732742063138Num uniques 859; 2024-11-27T21:14:23,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-27T21:14:23,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1732742063092, max timestamp=1732742063138Num uniques 859; 2024-11-27T21:14:23,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-27T21:14:23,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742123142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742123144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742123145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742123148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742123149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T21:14:23,220 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:23,220 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-27T21:14:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T21:14:23,223 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:23,223 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:23,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:23,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742123259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742123259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742123259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742123259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742123259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T21:14:23,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:14:23,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:23,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:23,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:23,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:23,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:23,460 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/f4ac37da4d8f4aab97b7ec8caf835cb1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/f4ac37da4d8f4aab97b7ec8caf835cb1 2024-11-27T21:14:23,466 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/A of 6d7b62f727fc76cea63daea76cc211d5 into f4ac37da4d8f4aab97b7ec8caf835cb1(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:23,466 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:23,466 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/A, priority=13, startTime=1732742062560; duration=0sec 2024-11-27T21:14:23,466 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:23,466 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:A 2024-11-27T21:14:23,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742123469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742123469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742123469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742123469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742123470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,513 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:23,516 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b97745bf5d9848b3874e20717cf9847a_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b97745bf5d9848b3874e20717cf9847a_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:23,517 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/ef51e87c6dd64c01b78cea9dfa0bd610, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:23,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/ef51e87c6dd64c01b78cea9dfa0bd610 is 175, key is test_row_0/A:col10/1732742063090/Put/seqid=0 2024-11-27T21:14:23,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741972_1148 (size=65323) 2024-11-27T21:14:23,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:14:23,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:23,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:23,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:23,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:23,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T21:14:23,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:14:23,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:23,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:23,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:23,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:23,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:23,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742123779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742123779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742123779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742123779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742123780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:23,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:14:23,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:23,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:23,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:23,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:23,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T21:14:23,923 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/ef51e87c6dd64c01b78cea9dfa0bd610 2024-11-27T21:14:23,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/399410278c5d466099be2a620d7c3023 is 50, key is test_row_0/B:col10/1732742063090/Put/seqid=0 2024-11-27T21:14:23,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741973_1149 (size=12001) 2024-11-27T21:14:23,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:14:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:23,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:23,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:23,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:23,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:24,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:14:24,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:24,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:24,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:24,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:24,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:24,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742124289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:24,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742124290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:24,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742124290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:24,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742124290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:24,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:24,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:24,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:24,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:14:24,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742124290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:24,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:24,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/399410278c5d466099be2a620d7c3023 2024-11-27T21:14:24,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/583225b06c4949f184129969d2f39235 is 50, key is test_row_0/C:col10/1732742063090/Put/seqid=0 2024-11-27T21:14:24,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741974_1150 (size=12001) 2024-11-27T21:14:24,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T21:14:24,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:14:24,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:24,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:24,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:24,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:24,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:24,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:14:24,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:24,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:24,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:24,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:24,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:24,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/583225b06c4949f184129969d2f39235 2024-11-27T21:14:24,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:14:24,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:24,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:24,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:24,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:24,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:24,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/ef51e87c6dd64c01b78cea9dfa0bd610 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/ef51e87c6dd64c01b78cea9dfa0bd610 2024-11-27T21:14:24,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/ef51e87c6dd64c01b78cea9dfa0bd610, entries=350, sequenceid=81, filesize=63.8 K 2024-11-27T21:14:24,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/399410278c5d466099be2a620d7c3023 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/399410278c5d466099be2a620d7c3023 2024-11-27T21:14:24,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/399410278c5d466099be2a620d7c3023, entries=150, sequenceid=81, filesize=11.7 K 2024-11-27T21:14:24,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/583225b06c4949f184129969d2f39235 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/583225b06c4949f184129969d2f39235 2024-11-27T21:14:24,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/583225b06c4949f184129969d2f39235, entries=150, sequenceid=81, filesize=11.7 K 2024-11-27T21:14:24,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for 6d7b62f727fc76cea63daea76cc211d5 in 1692ms, sequenceid=81, compaction requested=true 2024-11-27T21:14:24,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:24,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:14:24,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:24,784 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:24,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:14:24,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:24,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:14:24,784 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:24,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:24,785 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127336 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:24,785 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/A is initiating minor compaction (all files) 2024-11-27T21:14:24,785 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/A in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:24,785 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/f4ac37da4d8f4aab97b7ec8caf835cb1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eaa10a5ee58249439495be8b8f95bfa4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/ef51e87c6dd64c01b78cea9dfa0bd610] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=124.4 K 2024-11-27T21:14:24,785 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:24,785 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/f4ac37da4d8f4aab97b7ec8caf835cb1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eaa10a5ee58249439495be8b8f95bfa4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/ef51e87c6dd64c01b78cea9dfa0bd610] 2024-11-27T21:14:24,786 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:24,786 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/B is initiating minor compaction (all files) 2024-11-27T21:14:24,786 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/B in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:24,786 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/a6d3fe13062146d1ac507224b8c9efb8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/9f3a2656bfd44eaf89967422c94343c8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/399410278c5d466099be2a620d7c3023] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=35.3 K 2024-11-27T21:14:24,786 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting a6d3fe13062146d1ac507224b8c9efb8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732742059714 2024-11-27T21:14:24,787 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4ac37da4d8f4aab97b7ec8caf835cb1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732742059714 2024-11-27T21:14:24,787 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3a2656bfd44eaf89967422c94343c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732742060896 2024-11-27T21:14:24,787 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 399410278c5d466099be2a620d7c3023, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732742063087 2024-11-27T21:14:24,788 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaa10a5ee58249439495be8b8f95bfa4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732742060896 2024-11-27T21:14:24,789 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef51e87c6dd64c01b78cea9dfa0bd610, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732742063072 2024-11-27T21:14:24,797 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:24,802 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#B#compaction#130 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:24,802 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/eb8a2fc51b80417bbf31d298701476c4 is 50, key is test_row_0/B:col10/1732742063090/Put/seqid=0 2024-11-27T21:14:24,804 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127b60ad7f091b84a62ad066f18d5c30f2c_6d7b62f727fc76cea63daea76cc211d5 store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:24,806 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127b60ad7f091b84a62ad066f18d5c30f2c_6d7b62f727fc76cea63daea76cc211d5, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:24,807 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b60ad7f091b84a62ad066f18d5c30f2c_6d7b62f727fc76cea63daea76cc211d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:24,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741976_1152 (size=4469) 2024-11-27T21:14:24,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741975_1151 (size=12207) 2024-11-27T21:14:24,831 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/eb8a2fc51b80417bbf31d298701476c4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/eb8a2fc51b80417bbf31d298701476c4 2024-11-27T21:14:24,839 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/B of 6d7b62f727fc76cea63daea76cc211d5 into eb8a2fc51b80417bbf31d298701476c4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:24,839 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:24,839 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/B, priority=13, startTime=1732742064784; duration=0sec 2024-11-27T21:14:24,840 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:24,840 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:B 2024-11-27T21:14:24,840 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:24,841 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:24,841 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/C is initiating minor compaction (all files) 2024-11-27T21:14:24,841 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/C in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:24,841 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/0b88b1902a2248a79cefd87bd1514332, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/1d598808def341098605a26b746245b9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/583225b06c4949f184129969d2f39235] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=35.3 K 2024-11-27T21:14:24,842 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b88b1902a2248a79cefd87bd1514332, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732742059714 2024-11-27T21:14:24,842 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d598808def341098605a26b746245b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732742060896 2024-11-27T21:14:24,842 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 583225b06c4949f184129969d2f39235, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732742063087 2024-11-27T21:14:24,849 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#C#compaction#131 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:24,850 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/fc1c98e3915d4e7491f37bc34c231c93 is 50, key is test_row_0/C:col10/1732742063090/Put/seqid=0 2024-11-27T21:14:24,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741977_1153 (size=12207) 2024-11-27T21:14:24,874 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/fc1c98e3915d4e7491f37bc34c231c93 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/fc1c98e3915d4e7491f37bc34c231c93 2024-11-27T21:14:24,889 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/C of 6d7b62f727fc76cea63daea76cc211d5 into fc1c98e3915d4e7491f37bc34c231c93(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:24,889 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:24,889 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/C, priority=13, startTime=1732742064784; duration=0sec 2024-11-27T21:14:24,890 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:24,890 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:C 2024-11-27T21:14:24,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T21:14:24,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:24,912 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-11-27T21:14:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:24,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127390c92f1954445e68e3729013faf2b26_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742063143/Put/seqid=0 2024-11-27T21:14:24,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741978_1154 (size=12154) 2024-11-27T21:14:25,231 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#A#compaction#129 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:25,232 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/a0491ec01d36478683c07654a91e723d is 175, key is test_row_0/A:col10/1732742063090/Put/seqid=0 2024-11-27T21:14:25,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741979_1155 (size=31161) 2024-11-27T21:14:25,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:25,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:25,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742125312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742125313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742125314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742125315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742125315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:25,361 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127390c92f1954445e68e3729013faf2b26_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127390c92f1954445e68e3729013faf2b26_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:25,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/eeb67821368740d190e7bc3d6d9ca8b0, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:25,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/eeb67821368740d190e7bc3d6d9ca8b0 is 175, key is test_row_0/A:col10/1732742063143/Put/seqid=0 2024-11-27T21:14:25,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741980_1156 (size=30955) 2024-11-27T21:14:25,369 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/eeb67821368740d190e7bc3d6d9ca8b0 2024-11-27T21:14:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T21:14:25,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/ebfdf4ce3a37419587fb9f6276327891 is 50, key is test_row_0/B:col10/1732742063143/Put/seqid=0 2024-11-27T21:14:25,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742125419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741981_1157 (size=12001) 2024-11-27T21:14:25,423 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/ebfdf4ce3a37419587fb9f6276327891 2024-11-27T21:14:25,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742125429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742125429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/97b5d4bd2f4c4f37beae8c01fde3d632 is 50, key is test_row_0/C:col10/1732742063143/Put/seqid=0 2024-11-27T21:14:25,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742125439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742125439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741982_1158 (size=12001) 2024-11-27T21:14:25,445 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/97b5d4bd2f4c4f37beae8c01fde3d632 2024-11-27T21:14:25,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/eeb67821368740d190e7bc3d6d9ca8b0 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eeb67821368740d190e7bc3d6d9ca8b0 2024-11-27T21:14:25,462 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eeb67821368740d190e7bc3d6d9ca8b0, entries=150, sequenceid=110, filesize=30.2 K 2024-11-27T21:14:25,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/ebfdf4ce3a37419587fb9f6276327891 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/ebfdf4ce3a37419587fb9f6276327891 2024-11-27T21:14:25,478 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/ebfdf4ce3a37419587fb9f6276327891, entries=150, sequenceid=110, filesize=11.7 K 2024-11-27T21:14:25,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/97b5d4bd2f4c4f37beae8c01fde3d632 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/97b5d4bd2f4c4f37beae8c01fde3d632 2024-11-27T21:14:25,484 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/97b5d4bd2f4c4f37beae8c01fde3d632, entries=150, sequenceid=110, filesize=11.7 K 2024-11-27T21:14:25,485 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=53.67 KB/54960 for 6d7b62f727fc76cea63daea76cc211d5 in 573ms, sequenceid=110, compaction requested=false 2024-11-27T21:14:25,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:25,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:25,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-27T21:14:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-27T21:14:25,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-27T21:14:25,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2660 sec 2024-11-27T21:14:25,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.2760 sec 2024-11-27T21:14:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:25,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-27T21:14:25,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:25,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:25,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:25,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:25,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:25,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:25,658 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/a0491ec01d36478683c07654a91e723d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/a0491ec01d36478683c07654a91e723d 2024-11-27T21:14:25,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273aa7092ce1894f1bbc671fac2baa8b33_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742065309/Put/seqid=0 2024-11-27T21:14:25,670 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/A of 6d7b62f727fc76cea63daea76cc211d5 into a0491ec01d36478683c07654a91e723d(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:25,670 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:25,670 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/A, priority=13, startTime=1732742064784; duration=0sec 2024-11-27T21:14:25,670 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:25,670 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:A 2024-11-27T21:14:25,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741983_1159 (size=14594) 2024-11-27T21:14:25,682 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:25,688 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273aa7092ce1894f1bbc671fac2baa8b33_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273aa7092ce1894f1bbc671fac2baa8b33_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:25,690 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/d79913c07acb44e88644f0ed60b38da6, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:25,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/d79913c07acb44e88644f0ed60b38da6 is 175, key is test_row_0/A:col10/1732742065309/Put/seqid=0 2024-11-27T21:14:25,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742125688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742125690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742125691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742125692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742125694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741984_1160 (size=39549) 2024-11-27T21:14:25,723 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/d79913c07acb44e88644f0ed60b38da6 2024-11-27T21:14:25,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/0c5927ea01264064916a92ace70966fe is 50, key is test_row_0/B:col10/1732742065309/Put/seqid=0 2024-11-27T21:14:25,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741985_1161 (size=12001) 2024-11-27T21:14:25,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742125799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742125799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742125809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742125809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:25,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:25,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742125809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:25,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:25,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742126009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742126009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742126019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742126020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742126020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/0c5927ea01264064916a92ace70966fe 2024-11-27T21:14:26,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/6a5484bb55114b5ea79fda6454b38c48 is 50, key is test_row_0/C:col10/1732742065309/Put/seqid=0 2024-11-27T21:14:26,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741986_1162 (size=12001) 2024-11-27T21:14:26,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742126319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742126319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742126330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742126330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742126330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/6a5484bb55114b5ea79fda6454b38c48 2024-11-27T21:14:26,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/d79913c07acb44e88644f0ed60b38da6 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d79913c07acb44e88644f0ed60b38da6 2024-11-27T21:14:26,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d79913c07acb44e88644f0ed60b38da6, entries=200, sequenceid=122, filesize=38.6 K 2024-11-27T21:14:26,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/0c5927ea01264064916a92ace70966fe as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0c5927ea01264064916a92ace70966fe 2024-11-27T21:14:26,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0c5927ea01264064916a92ace70966fe, entries=150, sequenceid=122, filesize=11.7 K 2024-11-27T21:14:26,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/6a5484bb55114b5ea79fda6454b38c48 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/6a5484bb55114b5ea79fda6454b38c48 2024-11-27T21:14:26,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/6a5484bb55114b5ea79fda6454b38c48, entries=150, sequenceid=122, filesize=11.7 K 2024-11-27T21:14:26,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6d7b62f727fc76cea63daea76cc211d5 in 997ms, sequenceid=122, compaction requested=true 2024-11-27T21:14:26,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:26,632 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:26,634 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:26,634 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/A is initiating minor compaction (all files) 2024-11-27T21:14:26,634 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/A in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:26,634 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/a0491ec01d36478683c07654a91e723d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eeb67821368740d190e7bc3d6d9ca8b0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d79913c07acb44e88644f0ed60b38da6] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=99.3 K 2024-11-27T21:14:26,634 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:26,634 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/a0491ec01d36478683c07654a91e723d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eeb67821368740d190e7bc3d6d9ca8b0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d79913c07acb44e88644f0ed60b38da6] 2024-11-27T21:14:26,637 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0491ec01d36478683c07654a91e723d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732742063087 2024-11-27T21:14:26,640 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeb67821368740d190e7bc3d6d9ca8b0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732742063143 2024-11-27T21:14:26,642 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting d79913c07acb44e88644f0ed60b38da6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732742065309 2024-11-27T21:14:26,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:14:26,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:26,648 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:26,649 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:26,649 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/B is initiating minor compaction (all files) 2024-11-27T21:14:26,649 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/B in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:26,650 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/eb8a2fc51b80417bbf31d298701476c4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/ebfdf4ce3a37419587fb9f6276327891, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0c5927ea01264064916a92ace70966fe] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=35.4 K 2024-11-27T21:14:26,650 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting eb8a2fc51b80417bbf31d298701476c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732742063087 2024-11-27T21:14:26,652 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting ebfdf4ce3a37419587fb9f6276327891, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732742063143 2024-11-27T21:14:26,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:14:26,653 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c5927ea01264064916a92ace70966fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732742065309 2024-11-27T21:14:26,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:26,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:14:26,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:26,660 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:26,675 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#B#compaction#139 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:26,675 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/0d28a4eebab949c6a192fab37dbd44c8 is 50, key is test_row_0/B:col10/1732742065309/Put/seqid=0 2024-11-27T21:14:26,697 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411273033a273f8b043658c31786d8faa93bf_6d7b62f727fc76cea63daea76cc211d5 store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:26,699 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411273033a273f8b043658c31786d8faa93bf_6d7b62f727fc76cea63daea76cc211d5, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:26,699 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273033a273f8b043658c31786d8faa93bf_6d7b62f727fc76cea63daea76cc211d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:26,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741987_1163 (size=12309) 2024-11-27T21:14:26,758 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/0d28a4eebab949c6a192fab37dbd44c8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0d28a4eebab949c6a192fab37dbd44c8 2024-11-27T21:14:26,766 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/B of 6d7b62f727fc76cea63daea76cc211d5 into 0d28a4eebab949c6a192fab37dbd44c8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:26,767 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:26,767 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/B, priority=13, startTime=1732742066648; duration=0sec 2024-11-27T21:14:26,767 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:26,767 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:B 2024-11-27T21:14:26,767 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:26,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741988_1164 (size=4469) 2024-11-27T21:14:26,768 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:26,768 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/C is initiating minor compaction (all files) 2024-11-27T21:14:26,768 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/C in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:26,768 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/fc1c98e3915d4e7491f37bc34c231c93, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/97b5d4bd2f4c4f37beae8c01fde3d632, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/6a5484bb55114b5ea79fda6454b38c48] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=35.4 K 2024-11-27T21:14:26,769 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#A#compaction#138 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:26,769 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/d18294942b2345cb88ba69cf76541613 is 175, key is test_row_0/A:col10/1732742065309/Put/seqid=0 2024-11-27T21:14:26,770 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting fc1c98e3915d4e7491f37bc34c231c93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732742063087 2024-11-27T21:14:26,771 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 97b5d4bd2f4c4f37beae8c01fde3d632, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732742063143 2024-11-27T21:14:26,772 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a5484bb55114b5ea79fda6454b38c48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732742065309 2024-11-27T21:14:26,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741989_1165 (size=31263) 2024-11-27T21:14:26,783 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:26,783 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/1d4b9693ceb3487487f49552d29565ed is 50, key is test_row_0/C:col10/1732742065309/Put/seqid=0 2024-11-27T21:14:26,787 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/d18294942b2345cb88ba69cf76541613 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d18294942b2345cb88ba69cf76541613 2024-11-27T21:14:26,798 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/A of 6d7b62f727fc76cea63daea76cc211d5 into d18294942b2345cb88ba69cf76541613(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:26,798 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:26,798 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/A, priority=13, startTime=1732742066632; duration=0sec 2024-11-27T21:14:26,798 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:26,798 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:A 2024-11-27T21:14:26,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741990_1166 (size=12309) 2024-11-27T21:14:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:26,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-27T21:14:26,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:26,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:26,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:26,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:26,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:26,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:26,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742126849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742126852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272d3df4fe066b4d5b8dcec055d38bf92c_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742066832/Put/seqid=0 2024-11-27T21:14:26,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742126854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742126854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742126855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741991_1167 (size=17284) 2024-11-27T21:14:26,868 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:26,875 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272d3df4fe066b4d5b8dcec055d38bf92c_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272d3df4fe066b4d5b8dcec055d38bf92c_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:26,878 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/e5b418d945a54f4eb13a632c8e81dfd5, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:26,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/e5b418d945a54f4eb13a632c8e81dfd5 is 175, key is test_row_0/A:col10/1732742066832/Put/seqid=0 2024-11-27T21:14:26,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741992_1168 (size=48389) 2024-11-27T21:14:26,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742126959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742126969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742126969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:26,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742126969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:27,036 INFO [master/4585c39bed03:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-27T21:14:27,036 INFO [master/4585c39bed03:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-27T21:14:27,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742127169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:27,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:27,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742127179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:27,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742127179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:27,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:27,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:27,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:27,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742127190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:27,213 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/1d4b9693ceb3487487f49552d29565ed as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/1d4b9693ceb3487487f49552d29565ed 2024-11-27T21:14:27,224 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/C of 6d7b62f727fc76cea63daea76cc211d5 into 1d4b9693ceb3487487f49552d29565ed(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:27,224 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:27,224 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/C, priority=13, startTime=1732742066656; duration=0sec 2024-11-27T21:14:27,224 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:27,224 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:C 2024-11-27T21:14:27,290 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/e5b418d945a54f4eb13a632c8e81dfd5 2024-11-27T21:14:27,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/e14dc3dd40f749709af500f522046eb7 is 50, key is test_row_0/B:col10/1732742066832/Put/seqid=0 2024-11-27T21:14:27,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741993_1169 (size=12151) 2024-11-27T21:14:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T21:14:27,390 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:27,390 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-27T21:14:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T21:14:27,393 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:27,394 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:27,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:27,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742127479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:27,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:27,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:27,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:27,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742127489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:27,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742127489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T21:14:27,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742127499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:27,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:27,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:27,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T21:14:27,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:27,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:27,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:27,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:27,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:27,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T21:14:27,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:27,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:27,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:27,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:27,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T21:14:27,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/e14dc3dd40f749709af500f522046eb7 2024-11-27T21:14:27,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/2704d02f56cd4845afa1a20f979b1715 is 50, key is test_row_0/C:col10/1732742066832/Put/seqid=0 2024-11-27T21:14:27,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741994_1170 (size=12151) 2024-11-27T21:14:27,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/2704d02f56cd4845afa1a20f979b1715 2024-11-27T21:14:27,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/e5b418d945a54f4eb13a632c8e81dfd5 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/e5b418d945a54f4eb13a632c8e81dfd5 2024-11-27T21:14:27,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/e5b418d945a54f4eb13a632c8e81dfd5, entries=250, sequenceid=147, filesize=47.3 K 2024-11-27T21:14:27,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/e14dc3dd40f749709af500f522046eb7 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/e14dc3dd40f749709af500f522046eb7 2024-11-27T21:14:27,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/e14dc3dd40f749709af500f522046eb7, entries=150, sequenceid=147, filesize=11.9 K 2024-11-27T21:14:27,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/2704d02f56cd4845afa1a20f979b1715 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/2704d02f56cd4845afa1a20f979b1715 2024-11-27T21:14:27,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/2704d02f56cd4845afa1a20f979b1715, entries=150, sequenceid=147, filesize=11.9 K 2024-11-27T21:14:27,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 6d7b62f727fc76cea63daea76cc211d5 in 953ms, sequenceid=147, compaction requested=false 2024-11-27T21:14:27,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:27,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T21:14:27,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:27,861 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:14:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:27,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:27,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278aa062e3f9254eae86365782412e07a4_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742066851/Put/seqid=0 2024-11-27T21:14:27,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741995_1171 (size=12304) 2024-11-27T21:14:27,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742067939, max timestamp=1732742067969Num uniques 859; 2024-11-27T21:14:27,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:14:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T21:14:28,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742128010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742128019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742128056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742128056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742128057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742128159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742128169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742128169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742128169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:28,331 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278aa062e3f9254eae86365782412e07a4_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278aa062e3f9254eae86365782412e07a4_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:28,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/27711a7b286a483c8a15b151509b1e77, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:28,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/27711a7b286a483c8a15b151509b1e77 is 175, key is test_row_0/A:col10/1732742066851/Put/seqid=0 2024-11-27T21:14:28,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741996_1172 (size=31105) 2024-11-27T21:14:28,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742128369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742128379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742128379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742128379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T21:14:28,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742128679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742128689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742128689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:28,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742128689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:28,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:28,755 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/27711a7b286a483c8a15b151509b1e77 2024-11-27T21:14:28,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/360415b3ee38480a916a14d2e22741d6 is 50, key is test_row_0/B:col10/1732742066851/Put/seqid=0 2024-11-27T21:14:28,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741997_1173 (size=12151) 2024-11-27T21:14:28,772 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/360415b3ee38480a916a14d2e22741d6 2024-11-27T21:14:28,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/ebee4620d0c34b80befebc0df8d16886 is 50, key is test_row_0/C:col10/1732742066851/Put/seqid=0 2024-11-27T21:14:28,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741998_1174 (size=12151) 2024-11-27T21:14:29,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742129069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:29,075 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in 6d7b62f727fc76cea63daea76cc211d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:29,187 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/ebee4620d0c34b80befebc0df8d16886 2024-11-27T21:14:29,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742129189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:29,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/27711a7b286a483c8a15b151509b1e77 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/27711a7b286a483c8a15b151509b1e77 2024-11-27T21:14:29,197 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/27711a7b286a483c8a15b151509b1e77, entries=150, sequenceid=158, filesize=30.4 K 2024-11-27T21:14:29,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/360415b3ee38480a916a14d2e22741d6 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/360415b3ee38480a916a14d2e22741d6 2024-11-27T21:14:29,203 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/360415b3ee38480a916a14d2e22741d6, entries=150, sequenceid=158, filesize=11.9 K 2024-11-27T21:14:29,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/ebee4620d0c34b80befebc0df8d16886 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/ebee4620d0c34b80befebc0df8d16886 2024-11-27T21:14:29,209 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/ebee4620d0c34b80befebc0df8d16886, entries=150, sequenceid=158, filesize=11.9 K 2024-11-27T21:14:29,210 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 6d7b62f727fc76cea63daea76cc211d5 in 1350ms, sequenceid=158, compaction requested=true 2024-11-27T21:14:29,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=149.83 KB heapSize=441.41 KB 2024-11-27T21:14:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:29,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:29,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:29,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-27T21:14:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-27T21:14:29,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:29,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:29,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:29,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:29,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:29,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:29,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-27T21:14:29,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8230 sec 2024-11-27T21:14:29,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.8320 sec 2024-11-27T21:14:29,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742129219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:29,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:29,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742129222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:29,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742129227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:29,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127edbe1cd018ee486e8b9cc9417ca09c93_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742069211/Put/seqid=0 2024-11-27T21:14:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:29,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741999_1175 (size=14794) 2024-11-27T21:14:29,245 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,249 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127edbe1cd018ee486e8b9cc9417ca09c93_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127edbe1cd018ee486e8b9cc9417ca09c93_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:29,251 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/8f60321be366466f88e0743d27c93e51, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:29,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/8f60321be366466f88e0743d27c93e51 is 175, key is test_row_0/A:col10/1732742069211/Put/seqid=0 2024-11-27T21:14:29,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742000_1176 (size=39749) 2024-11-27T21:14:29,257 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/8f60321be366466f88e0743d27c93e51 2024-11-27T21:14:29,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/288b5e938ea6424282df2e118daeb1cb is 50, key is test_row_0/B:col10/1732742069211/Put/seqid=0 2024-11-27T21:14:29,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742001_1177 (size=12151) 2024-11-27T21:14:29,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742129329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:29,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:29,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742129339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:29,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:29,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742129339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:29,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:29,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:29,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742129539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:29,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T21:14:29,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:29,550 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742129550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:29,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742129550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:29,551 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-27T21:14:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T21:14:29,555 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:29,555 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:29,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T21:14:29,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/288b5e938ea6424282df2e118daeb1cb 2024-11-27T21:14:29,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T21:14:29,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:29,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:29,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:29,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:29,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:29,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/724545c186a1475face2a5553b13271d is 50, key is test_row_0/C:col10/1732742069211/Put/seqid=0 2024-11-27T21:14:29,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742002_1178 (size=12151) 2024-11-27T21:14:29,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/724545c186a1475face2a5553b13271d 2024-11-27T21:14:29,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/8f60321be366466f88e0743d27c93e51 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/8f60321be366466f88e0743d27c93e51 2024-11-27T21:14:29,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/8f60321be366466f88e0743d27c93e51, entries=200, sequenceid=189, filesize=38.8 K 2024-11-27T21:14:29,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/288b5e938ea6424282df2e118daeb1cb as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/288b5e938ea6424282df2e118daeb1cb 2024-11-27T21:14:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/288b5e938ea6424282df2e118daeb1cb, entries=150, sequenceid=189, filesize=11.9 K 2024-11-27T21:14:29,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/724545c186a1475face2a5553b13271d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/724545c186a1475face2a5553b13271d 2024-11-27T21:14:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/724545c186a1475face2a5553b13271d, entries=150, sequenceid=189, filesize=11.9 K 2024-11-27T21:14:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~156.54 KB/160300, heapSize ~461.28 KB/472352, currentSize=20.13 KB/20610 for 6d7b62f727fc76cea63daea76cc211d5 in 572ms, sequenceid=189, compaction requested=true 2024-11-27T21:14:29,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:29,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:14:29,785 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:14:29,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,785 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:14:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:14:29,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:14:29,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,788 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150506 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:14:29,788 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/A is initiating minor compaction (all files) 2024-11-27T21:14:29,788 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/A in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,788 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d18294942b2345cb88ba69cf76541613, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/e5b418d945a54f4eb13a632c8e81dfd5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/27711a7b286a483c8a15b151509b1e77, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/8f60321be366466f88e0743d27c93e51] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=147.0 K 2024-11-27T21:14:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,788 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,788 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d18294942b2345cb88ba69cf76541613, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/e5b418d945a54f4eb13a632c8e81dfd5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/27711a7b286a483c8a15b151509b1e77, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/8f60321be366466f88e0743d27c93e51] 2024-11-27T21:14:29,788 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:14:29,788 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/B is initiating minor compaction (all files) 2024-11-27T21:14:29,788 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/B in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:29,789 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0d28a4eebab949c6a192fab37dbd44c8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/e14dc3dd40f749709af500f522046eb7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/360415b3ee38480a916a14d2e22741d6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/288b5e938ea6424282df2e118daeb1cb] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=47.6 K 2024-11-27T21:14:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,789 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting d18294942b2345cb88ba69cf76541613, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732742065309 2024-11-27T21:14:29,789 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d28a4eebab949c6a192fab37dbd44c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732742065309 2024-11-27T21:14:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,790 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5b418d945a54f4eb13a632c8e81dfd5, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732742065687 2024-11-27T21:14:29,790 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting e14dc3dd40f749709af500f522046eb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732742065689 2024-11-27T21:14:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,790 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 360415b3ee38480a916a14d2e22741d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732742066844 2024-11-27T21:14:29,790 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27711a7b286a483c8a15b151509b1e77, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732742066844 2024-11-27T21:14:29,791 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 288b5e938ea6424282df2e118daeb1cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732742069211 2024-11-27T21:14:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,791 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f60321be366466f88e0743d27c93e51, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732742068008 2024-11-27T21:14:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,805 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,807 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#B#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:29,807 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411277a5b42fcfb8e4bb5986ad112e06814a0_6d7b62f727fc76cea63daea76cc211d5 store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:29,807 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/5c685578278d47c1a378e5eb1e481d3c is 50, key is test_row_0/B:col10/1732742069211/Put/seqid=0 2024-11-27T21:14:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,810 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411277a5b42fcfb8e4bb5986ad112e06814a0_6d7b62f727fc76cea63daea76cc211d5, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,810 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277a5b42fcfb8e4bb5986ad112e06814a0_6d7b62f727fc76cea63daea76cc211d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742003_1179 (size=12595) 2024-11-27T21:14:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T21:14:29,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:29,864 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-27T21:14:29,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:29,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:29,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:29,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:29,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:29,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:29,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,869 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/5c685578278d47c1a378e5eb1e481d3c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/5c685578278d47c1a378e5eb1e481d3c 2024-11-27T21:14:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T21:14:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,877 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/B of 6d7b62f727fc76cea63daea76cc211d5 into 5c685578278d47c1a378e5eb1e481d3c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,877 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:29,877 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/B, priority=12, startTime=1732742069785; duration=0sec 2024-11-27T21:14:29,877 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:29,877 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:B 2024-11-27T21:14:29,877 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:14:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,879 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:14:29,879 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/C is initiating minor compaction (all files) 2024-11-27T21:14:29,879 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/C in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:29,879 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/1d4b9693ceb3487487f49552d29565ed, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/2704d02f56cd4845afa1a20f979b1715, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/ebee4620d0c34b80befebc0df8d16886, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/724545c186a1475face2a5553b13271d] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=47.6 K 2024-11-27T21:14:29,880 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d4b9693ceb3487487f49552d29565ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732742065309 2024-11-27T21:14:29,880 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 2704d02f56cd4845afa1a20f979b1715, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732742065689 2024-11-27T21:14:29,881 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting ebee4620d0c34b80befebc0df8d16886, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732742066844 2024-11-27T21:14:29,881 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 724545c186a1475face2a5553b13271d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732742069211 2024-11-27T21:14:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:29,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742004_1180 (size=4469) 2024-11-27T21:14:29,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c0cd62f1a129418dba6f3d5e55f6131e_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742069216/Put/seqid=0 2024-11-27T21:14:29,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,894 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#C#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:29,894 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/ed7ab10a74f74481b07e4d66861d4f26 is 50, key is test_row_0/C:col10/1732742069211/Put/seqid=0 2024-11-27T21:14:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:29,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742005_1181 (size=9814) 2024-11-27T21:14:29,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742006_1182 (size=12595) 2024-11-27T21:14:29,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742069908, max timestamp=1732742069923Num uniques 859; 2024-11-27T21:14:29,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:14:29,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742069908, max timestamp=1732742069923Num uniques 859; 2024-11-27T21:14:29,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742069908, max timestamp=1732742069923Num uniques 859; 2024-11-27T21:14:29,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:14:29,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:14:29,934 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/ed7ab10a74f74481b07e4d66861d4f26 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/ed7ab10a74f74481b07e4d66861d4f26 2024-11-27T21:14:29,942 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/C of 6d7b62f727fc76cea63daea76cc211d5 into ed7ab10a74f74481b07e4d66861d4f26(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:29,942 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:29,943 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/C, priority=12, startTime=1732742069786; duration=0sec 2024-11-27T21:14:29,943 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:29,943 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:C 2024-11-27T21:14:29,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742129985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:29,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732742069960, max timestamp=1732742069985Num uniques 1118; 2024-11-27T21:14:29,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-27T21:14:29,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7681, totalHeapSize=17712, min timestamp=1732742069960, max timestamp=1732742069985Num uniques 1118; 2024-11-27T21:14:29,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51191, new segment data size=7681, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-27T21:14:29,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7683, totalHeapSize=17712, min timestamp=1732742069960, max timestamp=1732742069985Num uniques 1118; 2024-11-27T21:14:29,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51193, new segment data size=7683, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-27T21:14:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732742070021, max timestamp=1732742070045Num uniques 1027; 2024-11-27T21:14:30,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732742070021, max timestamp=1732742070045Num uniques 1027; 2024-11-27T21:14:30,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-27T21:14:30,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732742070021, max timestamp=1732742070045Num uniques 1027; 2024-11-27T21:14:30,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-27T21:14:30,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-27T21:14:30,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732742070122, max timestamp=1732742070149Num uniques 1086; 2024-11-27T21:14:30,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732742070122, max timestamp=1732742070149Num uniques 1086; 2024-11-27T21:14:30,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-27T21:14:30,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-27T21:14:30,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8508, totalHeapSize=19584, min timestamp=1732742070122, max timestamp=1732742070150Num uniques 1086; 2024-11-27T21:14:30,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49728, new segment data size=8508, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-27T21:14:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T21:14:30,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1732742070166, max timestamp=1732742070212Num uniques 1145; 2024-11-27T21:14:30,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=18090, suffix heap size=147048,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=395 2024-11-27T21:14:30,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1732742070166, max timestamp=1732742070212Num uniques 1045; 2024-11-27T21:14:30,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=18090, suffix heap size=136648,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=395 2024-11-27T21:14:30,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1732742070166, max timestamp=1732742070212Num uniques 1145; 2024-11-27T21:14:30,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=18090, suffix heap size=147048,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=395 2024-11-27T21:14:30,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:30,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742130254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742130256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742130267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:30,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742130267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,288 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#A#compaction#150 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:30,289 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/f27c49edfebb4ddea006425e0a592ac9 is 175, key is test_row_0/A:col10/1732742069211/Put/seqid=0 2024-11-27T21:14:30,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742007_1183 (size=31549) 2024-11-27T21:14:30,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:30,327 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c0cd62f1a129418dba6f3d5e55f6131e_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c0cd62f1a129418dba6f3d5e55f6131e_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:30,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/9cfcad38864a4a8baa80d76cb0f9f1c1, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/9cfcad38864a4a8baa80d76cb0f9f1c1 is 175, key is test_row_0/A:col10/1732742069216/Put/seqid=0 2024-11-27T21:14:30,330 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/f27c49edfebb4ddea006425e0a592ac9 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/f27c49edfebb4ddea006425e0a592ac9 2024-11-27T21:14:30,338 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/A of 6d7b62f727fc76cea63daea76cc211d5 into f27c49edfebb4ddea006425e0a592ac9(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:30,338 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:30,338 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/A, priority=12, startTime=1732742069785; duration=0sec 2024-11-27T21:14:30,338 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:30,338 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:A 2024-11-27T21:14:30,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742008_1184 (size=22461) 2024-11-27T21:14:30,382 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/9cfcad38864a4a8baa80d76cb0f9f1c1 2024-11-27T21:14:30,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742130379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742130379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:30,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742130381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742130381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/a076b91d35ad4eca81147b60b65e5ad1 is 50, key is test_row_0/B:col10/1732742069216/Put/seqid=0 2024-11-27T21:14:30,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742009_1185 (size=9757) 2024-11-27T21:14:30,446 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/a076b91d35ad4eca81147b60b65e5ad1 2024-11-27T21:14:30,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/7f0653e5ed2c4aabb2d2f69fc89a71b3 is 50, key is test_row_0/C:col10/1732742069216/Put/seqid=0 2024-11-27T21:14:30,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742010_1186 (size=9757) 2024-11-27T21:14:30,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742130589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:30,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742130589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742130589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:30,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742130599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T21:14:30,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742130899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742130899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742130902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,911 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/7f0653e5ed2c4aabb2d2f69fc89a71b3 2024-11-27T21:14:30,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742130909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:30,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:30,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:30,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/9cfcad38864a4a8baa80d76cb0f9f1c1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/9cfcad38864a4a8baa80d76cb0f9f1c1 2024-11-27T21:14:30,929 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/9cfcad38864a4a8baa80d76cb0f9f1c1, entries=100, sequenceid=195, filesize=21.9 K 2024-11-27T21:14:30,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/a076b91d35ad4eca81147b60b65e5ad1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/a076b91d35ad4eca81147b60b65e5ad1 2024-11-27T21:14:30,936 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/a076b91d35ad4eca81147b60b65e5ad1, entries=100, sequenceid=195, filesize=9.5 K 2024-11-27T21:14:30,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/7f0653e5ed2c4aabb2d2f69fc89a71b3 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/7f0653e5ed2c4aabb2d2f69fc89a71b3 2024-11-27T21:14:30,943 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/7f0653e5ed2c4aabb2d2f69fc89a71b3, entries=100, sequenceid=195, filesize=9.5 K 2024-11-27T21:14:30,944 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=167.72 KB/171750 for 6d7b62f727fc76cea63daea76cc211d5 in 1080ms, sequenceid=195, compaction requested=false 2024-11-27T21:14:30,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:30,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:30,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-27T21:14:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-27T21:14:30,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-27T21:14:30,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3900 sec 2024-11-27T21:14:30,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.3970 sec 2024-11-27T21:14:31,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732742070252, max timestamp=1732742071089Num uniques 1254; 2024-11-27T21:14:31,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732742070252, max timestamp=1732742071089Num uniques 1254; 2024-11-27T21:14:31,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-11-27T21:14:31,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732742070252, max timestamp=1732742071089Num uniques 1254; 2024-11-27T21:14:31,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-11-27T21:14:31,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-11-27T21:14:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:31,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=53.67 KB heapSize=146.39 KB 2024-11-27T21:14:31,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:31,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:31,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:31,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:31,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:31,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:31,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275e6107f5f54547cca2f4223e2508dd96_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742071103/Put/seqid=0 2024-11-27T21:14:31,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742011_1187 (size=14994) 2024-11-27T21:14:31,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742071181, max timestamp=1732742071260Num uniques 859; 2024-11-27T21:14:31,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:14:31,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742071181, max timestamp=1732742071260Num uniques 859; 2024-11-27T21:14:31,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:14:31,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742071181, max timestamp=1732742071260Num uniques 859; 2024-11-27T21:14:31,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:14:31,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742131361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:31,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:31,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:31,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:31,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742131409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:31,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:31,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:31,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:31,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742131409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:31,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:31,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:31,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:31,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742131419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:31,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742131419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:31,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742131479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:31,549 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:31,554 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275e6107f5f54547cca2f4223e2508dd96_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275e6107f5f54547cca2f4223e2508dd96_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:31,555 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/4c560bc949534343a2377c8008aa0857, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:31,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/4c560bc949534343a2377c8008aa0857 is 175, key is test_row_0/A:col10/1732742071103/Put/seqid=0 2024-11-27T21:14:31,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742012_1188 (size=39949) 2024-11-27T21:14:31,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742131689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T21:14:31,699 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:31,700 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-27T21:14:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T21:14:31,702 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:31,702 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:31,702 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T21:14:31,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T21:14:31,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:31,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:31,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:31,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:31,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:31,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:31,961 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/4c560bc949534343a2377c8008aa0857 2024-11-27T21:14:31,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/2ddc420d5e024fd694233035df99a2bc is 50, key is test_row_0/B:col10/1732742071103/Put/seqid=0 2024-11-27T21:14:31,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742013_1189 (size=12301) 2024-11-27T21:14:31,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/2ddc420d5e024fd694233035df99a2bc 2024-11-27T21:14:32,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:32,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742131999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:32,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:32,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:32,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:32,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/3b8091ec43e04a6b99cb465218bf6e5c is 50, key is test_row_0/C:col10/1732742071103/Put/seqid=0 2024-11-27T21:14:32,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T21:14:32,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:32,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:32,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:32,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:32,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:32,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:32,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742014_1190 (size=12301) 2024-11-27T21:14:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T21:14:32,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T21:14:32,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:32,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:32,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:32,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:32,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:32,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T21:14:32,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:32,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:32,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:32,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:32,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:32,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T21:14:32,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/3b8091ec43e04a6b99cb465218bf6e5c 2024-11-27T21:14:32,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/4c560bc949534343a2377c8008aa0857 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/4c560bc949534343a2377c8008aa0857 2024-11-27T21:14:32,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:32,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742132430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:32,440 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in 6d7b62f727fc76cea63daea76cc211d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:32,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:32,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/4c560bc949534343a2377c8008aa0857, entries=200, sequenceid=309, filesize=39.0 K 2024-11-27T21:14:32,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/2ddc420d5e024fd694233035df99a2bc as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/2ddc420d5e024fd694233035df99a2bc 2024-11-27T21:14:32,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/2ddc420d5e024fd694233035df99a2bc, entries=150, sequenceid=309, filesize=12.0 K 2024-11-27T21:14:32,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742132436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:32,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742132436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:32,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/3b8091ec43e04a6b99cb465218bf6e5c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/3b8091ec43e04a6b99cb465218bf6e5c 2024-11-27T21:14:32,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742132439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:32,449 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_0' in 6d7b62f727fc76cea63daea76cc211d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:32,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:32,450 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in 6d7b62f727fc76cea63daea76cc211d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:32,450 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in 6d7b62f727fc76cea63daea76cc211d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:32,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:32,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:32,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:32,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:32,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:32,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:32,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/3b8091ec43e04a6b99cb465218bf6e5c, entries=150, sequenceid=309, filesize=12.0 K 2024-11-27T21:14:32,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~146.34 KB/149856, currentSize=134.18 KB/137400 for 6d7b62f727fc76cea63daea76cc211d5 in 1344ms, sequenceid=309, compaction requested=true 2024-11-27T21:14:32,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:32,456 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:32,457 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:32,457 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/A is initiating minor compaction (all files) 2024-11-27T21:14:32,457 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/A in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:32,458 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/f27c49edfebb4ddea006425e0a592ac9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/9cfcad38864a4a8baa80d76cb0f9f1c1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/4c560bc949534343a2377c8008aa0857] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=91.8 K 2024-11-27T21:14:32,458 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:32,458 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/f27c49edfebb4ddea006425e0a592ac9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/9cfcad38864a4a8baa80d76cb0f9f1c1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/4c560bc949534343a2377c8008aa0857] 2024-11-27T21:14:32,458 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting f27c49edfebb4ddea006425e0a592ac9, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732742069211 2024-11-27T21:14:32,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:14:32,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:32,459 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cfcad38864a4a8baa80d76cb0f9f1c1, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732742069216 2024-11-27T21:14:32,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:14:32,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:32,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:14:32,459 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:32,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:32,460 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c560bc949534343a2377c8008aa0857, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732742071089 2024-11-27T21:14:32,461 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:32,461 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/B is initiating minor compaction (all files) 2024-11-27T21:14:32,461 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/B in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:32,461 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/5c685578278d47c1a378e5eb1e481d3c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/a076b91d35ad4eca81147b60b65e5ad1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/2ddc420d5e024fd694233035df99a2bc] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=33.8 K 2024-11-27T21:14:32,462 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c685578278d47c1a378e5eb1e481d3c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732742069211 2024-11-27T21:14:32,463 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting a076b91d35ad4eca81147b60b65e5ad1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732742069216 2024-11-27T21:14:32,463 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ddc420d5e024fd694233035df99a2bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732742071103 2024-11-27T21:14:32,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T21:14:32,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:32,471 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-27T21:14:32,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:32,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:32,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:32,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:32,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:32,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:32,477 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:32,489 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127ebe834ca251f47808f9abbaf807597ca_6d7b62f727fc76cea63daea76cc211d5 store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:32,490 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#B#compaction#160 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:32,491 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/abfd9d8390dd43549d1bfcd3e5a81a8b is 50, key is test_row_0/B:col10/1732742071103/Put/seqid=0 2024-11-27T21:14:32,492 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127ebe834ca251f47808f9abbaf807597ca_6d7b62f727fc76cea63daea76cc211d5, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:32,492 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ebe834ca251f47808f9abbaf807597ca_6d7b62f727fc76cea63daea76cc211d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:32,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ee84e0b14988466d931da105b50eed70_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742071350/Put/seqid=0 2024-11-27T21:14:32,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742016_1192 (size=12454) 2024-11-27T21:14:32,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:32,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742015_1191 (size=12847) 2024-11-27T21:14:32,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:32,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:32,518 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ee84e0b14988466d931da105b50eed70_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ee84e0b14988466d931da105b50eed70_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:32,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/3b483cd725384ad5899f9b6233f9da25, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:32,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/3b483cd725384ad5899f9b6233f9da25 is 175, key is test_row_0/A:col10/1732742071350/Put/seqid=0 2024-11-27T21:14:32,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742017_1193 (size=4469) 2024-11-27T21:14:32,528 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#A#compaction#159 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:32,528 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/a32009503d8f42099bf90597e51158e2 is 175, key is test_row_0/A:col10/1732742071103/Put/seqid=0 2024-11-27T21:14:32,531 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/abfd9d8390dd43549d1bfcd3e5a81a8b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/abfd9d8390dd43549d1bfcd3e5a81a8b 2024-11-27T21:14:32,541 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/B of 6d7b62f727fc76cea63daea76cc211d5 into abfd9d8390dd43549d1bfcd3e5a81a8b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:32,541 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:32,541 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/B, priority=13, startTime=1732742072459; duration=0sec 2024-11-27T21:14:32,547 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:32,547 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:B 2024-11-27T21:14:32,547 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:32,549 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:32,551 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/C is initiating minor compaction (all files) 2024-11-27T21:14:32,551 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/C in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:32,551 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/ed7ab10a74f74481b07e4d66861d4f26, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/7f0653e5ed2c4aabb2d2f69fc89a71b3, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/3b8091ec43e04a6b99cb465218bf6e5c] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=33.8 K 2024-11-27T21:14:32,552 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting ed7ab10a74f74481b07e4d66861d4f26, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732742069211 2024-11-27T21:14:32,552 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f0653e5ed2c4aabb2d2f69fc89a71b3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732742069216 2024-11-27T21:14:32,553 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b8091ec43e04a6b99cb465218bf6e5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732742071103 2024-11-27T21:14:32,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742018_1194 (size=31255) 2024-11-27T21:14:32,556 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/3b483cd725384ad5899f9b6233f9da25 2024-11-27T21:14:32,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742019_1195 (size=31801) 2024-11-27T21:14:32,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742132572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:32,587 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/a32009503d8f42099bf90597e51158e2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/a32009503d8f42099bf90597e51158e2 2024-11-27T21:14:32,589 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#C#compaction#162 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:32,589 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/699051159bd349e8a7c2253476cef902 is 50, key is test_row_0/C:col10/1732742071103/Put/seqid=0 2024-11-27T21:14:32,593 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/A of 6d7b62f727fc76cea63daea76cc211d5 into a32009503d8f42099bf90597e51158e2(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:32,593 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:32,593 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/A, priority=13, startTime=1732742072456; duration=0sec 2024-11-27T21:14:32,594 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:32,594 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:A 2024-11-27T21:14:32,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/b597303c06364a80aec6d80fdd105511 is 50, key is test_row_0/B:col10/1732742071350/Put/seqid=0 2024-11-27T21:14:32,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742020_1196 (size=12847) 2024-11-27T21:14:32,618 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/699051159bd349e8a7c2253476cef902 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/699051159bd349e8a7c2253476cef902 2024-11-27T21:14:32,626 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/C of 6d7b62f727fc76cea63daea76cc211d5 into 699051159bd349e8a7c2253476cef902(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:32,626 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:32,626 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/C, priority=13, startTime=1732742072459; duration=0sec 2024-11-27T21:14:32,627 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:32,627 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:C 2024-11-27T21:14:32,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742021_1197 (size=12301) 2024-11-27T21:14:32,631 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/b597303c06364a80aec6d80fdd105511 2024-11-27T21:14:32,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/683b38c1775c4b28904bd0d9f2450d1e is 50, key is test_row_0/C:col10/1732742071350/Put/seqid=0 2024-11-27T21:14:32,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742022_1198 (size=12301) 2024-11-27T21:14:32,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:32,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742132679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T21:14:32,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742132889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:33,069 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/683b38c1775c4b28904bd0d9f2450d1e 2024-11-27T21:14:33,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/3b483cd725384ad5899f9b6233f9da25 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/3b483cd725384ad5899f9b6233f9da25 2024-11-27T21:14:33,080 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/3b483cd725384ad5899f9b6233f9da25, entries=150, sequenceid=345, filesize=30.5 K 2024-11-27T21:14:33,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/b597303c06364a80aec6d80fdd105511 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/b597303c06364a80aec6d80fdd105511 2024-11-27T21:14:33,096 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/b597303c06364a80aec6d80fdd105511, entries=150, sequenceid=345, filesize=12.0 K 2024-11-27T21:14:33,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/683b38c1775c4b28904bd0d9f2450d1e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/683b38c1775c4b28904bd0d9f2450d1e 2024-11-27T21:14:33,111 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/683b38c1775c4b28904bd0d9f2450d1e, entries=150, sequenceid=345, filesize=12.0 K 2024-11-27T21:14:33,112 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6d7b62f727fc76cea63daea76cc211d5 in 641ms, sequenceid=345, compaction requested=false 2024-11-27T21:14:33,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:33,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:33,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-27T21:14:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-27T21:14:33,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-27T21:14:33,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-27T21:14:33,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4170 sec 2024-11-27T21:14:33,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:33,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-27T21:14:33,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:33,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:33,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:33,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:33,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:33,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:33,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112725f8f0f9ad344d65b4cd867c543df140_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742072541/Put/seqid=0 2024-11-27T21:14:33,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742023_1199 (size=12454) 2024-11-27T21:14:33,247 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:33,253 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112725f8f0f9ad344d65b4cd867c543df140_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112725f8f0f9ad344d65b4cd867c543df140_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:33,254 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/0adc7986b5c54cc8a1530de98a6160a2, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:33,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/0adc7986b5c54cc8a1530de98a6160a2 is 175, key is test_row_0/A:col10/1732742072541/Put/seqid=0 2024-11-27T21:14:33,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742024_1200 (size=31255) 2024-11-27T21:14:33,273 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/0adc7986b5c54cc8a1530de98a6160a2 2024-11-27T21:14:33,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/f6f4e7ac71424ca59a572cfd48bd94bc is 50, key is test_row_0/B:col10/1732742072541/Put/seqid=0 2024-11-27T21:14:33,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742073274, max timestamp=1732742073306Num uniques 859; 2024-11-27T21:14:33,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742073274, max timestamp=1732742073306Num uniques 859; 2024-11-27T21:14:33,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:14:33,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742073274, max timestamp=1732742073306Num uniques 859; 2024-11-27T21:14:33,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:14:33,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:14:33,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742025_1201 (size=12301) 2024-11-27T21:14:33,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/f6f4e7ac71424ca59a572cfd48bd94bc 2024-11-27T21:14:33,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/3743637b6e1346ffaeb08b2cdf803ea3 is 50, key is test_row_0/C:col10/1732742072541/Put/seqid=0 2024-11-27T21:14:33,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742026_1202 (size=12301) 2024-11-27T21:14:33,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:33,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742133429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:33,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:33,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:33,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742133549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:33,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/3743637b6e1346ffaeb08b2cdf803ea3 2024-11-27T21:14:33,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/0adc7986b5c54cc8a1530de98a6160a2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0adc7986b5c54cc8a1530de98a6160a2 2024-11-27T21:14:33,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0adc7986b5c54cc8a1530de98a6160a2, entries=150, sequenceid=359, filesize=30.5 K 2024-11-27T21:14:33,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/f6f4e7ac71424ca59a572cfd48bd94bc as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/f6f4e7ac71424ca59a572cfd48bd94bc 2024-11-27T21:14:33,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:33,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742133759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:33,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/f6f4e7ac71424ca59a572cfd48bd94bc, entries=150, sequenceid=359, filesize=12.0 K 2024-11-27T21:14:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:33,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:33,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:33,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/3743637b6e1346ffaeb08b2cdf803ea3 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/3743637b6e1346ffaeb08b2cdf803ea3 2024-11-27T21:14:33,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/3743637b6e1346ffaeb08b2cdf803ea3, entries=150, sequenceid=359, filesize=12.0 K 2024-11-27T21:14:33,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6d7b62f727fc76cea63daea76cc211d5 in 589ms, sequenceid=359, compaction requested=true 2024-11-27T21:14:33,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:33,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:14:33,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:33,789 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:33,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:14:33,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:33,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:14:33,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:33,790 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:33,791 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:33,791 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/A is initiating minor compaction (all files) 2024-11-27T21:14:33,791 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/A in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:33,791 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:33,791 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/a32009503d8f42099bf90597e51158e2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/3b483cd725384ad5899f9b6233f9da25, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0adc7986b5c54cc8a1530de98a6160a2] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=92.1 K 2024-11-27T21:14:33,791 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/B is initiating minor compaction (all files) 2024-11-27T21:14:33,791 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:33,791 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/a32009503d8f42099bf90597e51158e2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/3b483cd725384ad5899f9b6233f9da25, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0adc7986b5c54cc8a1530de98a6160a2] 2024-11-27T21:14:33,791 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/B in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:33,791 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/abfd9d8390dd43549d1bfcd3e5a81a8b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/b597303c06364a80aec6d80fdd105511, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/f6f4e7ac71424ca59a572cfd48bd94bc] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=36.6 K 2024-11-27T21:14:33,791 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting a32009503d8f42099bf90597e51158e2, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732742071103 2024-11-27T21:14:33,791 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting abfd9d8390dd43549d1bfcd3e5a81a8b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732742071103 2024-11-27T21:14:33,792 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b483cd725384ad5899f9b6233f9da25, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732742071336 2024-11-27T21:14:33,792 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting b597303c06364a80aec6d80fdd105511, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732742071336 2024-11-27T21:14:33,792 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0adc7986b5c54cc8a1530de98a6160a2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732742072541 2024-11-27T21:14:33,792 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting f6f4e7ac71424ca59a572cfd48bd94bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732742072541 2024-11-27T21:14:33,819 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:33,819 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/ace608040c564b1a9febaaa7728434f1 is 50, key is test_row_0/B:col10/1732742072541/Put/seqid=0 2024-11-27T21:14:33,829 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:33,833 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411278da3743a1f79446fb75bb32f735ce84d_6d7b62f727fc76cea63daea76cc211d5 store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:33,835 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411278da3743a1f79446fb75bb32f735ce84d_6d7b62f727fc76cea63daea76cc211d5, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:33,835 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278da3743a1f79446fb75bb32f735ce84d_6d7b62f727fc76cea63daea76cc211d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:33,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742027_1203 (size=12949) 2024-11-27T21:14:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T21:14:33,850 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:33,850 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-27T21:14:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T21:14:33,853 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:33,853 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:33,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:33,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742028_1204 (size=4469) 2024-11-27T21:14:33,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T21:14:34,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T21:14:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:34,006 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-27T21:14:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:34,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:34,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:34,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:34,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:34,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127416914196145421292e336054a284228_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742073423/Put/seqid=0 2024-11-27T21:14:34,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742029_1205 (size=12454) 2024-11-27T21:14:34,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,032 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127416914196145421292e336054a284228_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127416914196145421292e336054a284228_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:34,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/713b6b06ad45438cae391cecb2f2d090, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:34,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/713b6b06ad45438cae391cecb2f2d090 is 175, key is test_row_0/A:col10/1732742073423/Put/seqid=0 2024-11-27T21:14:34,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742030_1206 (size=31255) 2024-11-27T21:14:34,040 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/713b6b06ad45438cae391cecb2f2d090 2024-11-27T21:14:34,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/77c0fc26a65745d9ba7ea6441336b5b2 is 50, key is test_row_0/B:col10/1732742073423/Put/seqid=0 2024-11-27T21:14:34,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742031_1207 (size=12301) 2024-11-27T21:14:34,056 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/77c0fc26a65745d9ba7ea6441336b5b2 2024-11-27T21:14:34,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/e75e4ba026fc488e85da2737b2938a67 is 50, key is test_row_0/C:col10/1732742073423/Put/seqid=0 2024-11-27T21:14:34,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742032_1208 (size=12301) 2024-11-27T21:14:34,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:34,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:34,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742134133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:34,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T21:14:34,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:34,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742134239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:34,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:34,254 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/ace608040c564b1a9febaaa7728434f1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/ace608040c564b1a9febaaa7728434f1 2024-11-27T21:14:34,263 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#A#compaction#169 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:34,265 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/B of 6d7b62f727fc76cea63daea76cc211d5 into ace608040c564b1a9febaaa7728434f1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:34,265 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:34,265 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/B, priority=13, startTime=1732742073789; duration=0sec 2024-11-27T21:14:34,265 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:34,265 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:B 2024-11-27T21:14:34,265 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:34,267 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/bd4eb98dfbdf461191bb41490a15390b is 175, key is test_row_0/A:col10/1732742072541/Put/seqid=0 2024-11-27T21:14:34,269 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:34,269 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/C is initiating minor compaction (all files) 2024-11-27T21:14:34,269 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/C in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:34,269 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/699051159bd349e8a7c2253476cef902, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/683b38c1775c4b28904bd0d9f2450d1e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/3743637b6e1346ffaeb08b2cdf803ea3] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=36.6 K 2024-11-27T21:14:34,270 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 699051159bd349e8a7c2253476cef902, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732742071103 2024-11-27T21:14:34,274 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 683b38c1775c4b28904bd0d9f2450d1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732742071336 2024-11-27T21:14:34,275 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 3743637b6e1346ffaeb08b2cdf803ea3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732742072541 2024-11-27T21:14:34,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742033_1209 (size=31903) 2024-11-27T21:14:34,327 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#C#compaction#173 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:34,328 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/bf1444b25d07456b953ea9c28f284c2f is 50, key is test_row_0/C:col10/1732742072541/Put/seqid=0 2024-11-27T21:14:34,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742034_1210 (size=12949) 2024-11-27T21:14:34,348 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/bf1444b25d07456b953ea9c28f284c2f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/bf1444b25d07456b953ea9c28f284c2f 2024-11-27T21:14:34,356 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/C of 6d7b62f727fc76cea63daea76cc211d5 into bf1444b25d07456b953ea9c28f284c2f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:34,356 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:34,356 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/C, priority=13, startTime=1732742073789; duration=0sec 2024-11-27T21:14:34,356 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:34,356 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:C 2024-11-27T21:14:34,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742134449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:34,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742134459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:34,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742134459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:34,460 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_0' in 6d7b62f727fc76cea63daea76cc211d5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:34,461 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in 6d7b62f727fc76cea63daea76cc211d5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:34,473 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/e75e4ba026fc488e85da2737b2938a67 2024-11-27T21:14:34,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742134469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:34,480 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in 6d7b62f727fc76cea63daea76cc211d5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:34,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:34,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742134480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:34,481 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in 6d7b62f727fc76cea63daea76cc211d5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:34,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/713b6b06ad45438cae391cecb2f2d090 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/713b6b06ad45438cae391cecb2f2d090 2024-11-27T21:14:34,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T21:14:34,492 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/713b6b06ad45438cae391cecb2f2d090, entries=150, sequenceid=394, filesize=30.5 K 2024-11-27T21:14:34,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/77c0fc26a65745d9ba7ea6441336b5b2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/77c0fc26a65745d9ba7ea6441336b5b2 2024-11-27T21:14:34,499 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/77c0fc26a65745d9ba7ea6441336b5b2, entries=150, sequenceid=394, filesize=12.0 K 2024-11-27T21:14:34,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/e75e4ba026fc488e85da2737b2938a67 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e75e4ba026fc488e85da2737b2938a67 2024-11-27T21:14:34,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,514 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e75e4ba026fc488e85da2737b2938a67, entries=150, sequenceid=394, filesize=12.0 K 2024-11-27T21:14:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,516 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 6d7b62f727fc76cea63daea76cc211d5 in 510ms, sequenceid=394, compaction requested=false 2024-11-27T21:14:34,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:34,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:34,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-27T21:14:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-27T21:14:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-27T21:14:34,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 665 msec 2024-11-27T21:14:34,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 671 msec 2024-11-27T21:14:34,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,721 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/bd4eb98dfbdf461191bb41490a15390b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/bd4eb98dfbdf461191bb41490a15390b 2024-11-27T21:14:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,728 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/A of 6d7b62f727fc76cea63daea76cc211d5 into bd4eb98dfbdf461191bb41490a15390b(size=31.2 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:34,728 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:34,728 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/A, priority=13, startTime=1732742073789; duration=0sec 2024-11-27T21:14:34,728 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:34,728 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:A 2024-11-27T21:14:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-27T21:14:34,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:34,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:34,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:34,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:34,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:34,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275693ad8af85c4deb86b55f912c3ae64e_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742074131/Put/seqid=0 2024-11-27T21:14:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742035_1211 (size=12454) 2024-11-27T21:14:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,815 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,821 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275693ad8af85c4deb86b55f912c3ae64e_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275693ad8af85c4deb86b55f912c3ae64e_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:34,822 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/0053a8924cee46a8815c46a772e78752, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:34,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/0053a8924cee46a8815c46a772e78752 is 175, key is test_row_0/A:col10/1732742074131/Put/seqid=0 2024-11-27T21:14:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:34,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742036_1212 (size=31251) 2024-11-27T21:14:34,856 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/0053a8924cee46a8815c46a772e78752 2024-11-27T21:14:34,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/0043a8c5a037476f90046cba488e5863 is 50, key is test_row_0/B:col10/1732742074131/Put/seqid=0 2024-11-27T21:14:34,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742037_1213 (size=9857) 2024-11-27T21:14:34,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742134916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T21:14:34,999 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:35,000 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-27T21:14:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T21:14:35,002 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:35,002 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:35,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:35,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742135029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:35,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:35,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:35,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T21:14:35,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T21:14:35,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:35,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:35,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:35,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:35,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:35,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742135239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:35,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/0043a8c5a037476f90046cba488e5863 2024-11-27T21:14:35,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/53d8ab0f7992498b8d606d077534bed4 is 50, key is test_row_0/C:col10/1732742074131/Put/seqid=0 2024-11-27T21:14:35,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742038_1214 (size=9857) 2024-11-27T21:14:35,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T21:14:35,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:35,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:35,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:35,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:35,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T21:14:35,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T21:14:35,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:35,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:35,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742135549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:35,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T21:14:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:35,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T21:14:35,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/53d8ab0f7992498b8d606d077534bed4 2024-11-27T21:14:35,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/0053a8924cee46a8815c46a772e78752 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0053a8924cee46a8815c46a772e78752 2024-11-27T21:14:35,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0053a8924cee46a8815c46a772e78752, entries=150, sequenceid=409, filesize=30.5 K 2024-11-27T21:14:35,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/0043a8c5a037476f90046cba488e5863 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0043a8c5a037476f90046cba488e5863 2024-11-27T21:14:35,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0043a8c5a037476f90046cba488e5863, entries=100, sequenceid=409, filesize=9.6 K 2024-11-27T21:14:35,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/53d8ab0f7992498b8d606d077534bed4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/53d8ab0f7992498b8d606d077534bed4 2024-11-27T21:14:35,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/53d8ab0f7992498b8d606d077534bed4, entries=100, sequenceid=409, filesize=9.6 K 2024-11-27T21:14:35,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6d7b62f727fc76cea63daea76cc211d5 in 956ms, sequenceid=409, compaction requested=true 2024-11-27T21:14:35,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:35,722 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:35,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:14:35,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:35,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:14:35,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:35,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:14:35,722 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:35,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:35,723 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:35,723 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/A is initiating minor compaction (all files) 2024-11-27T21:14:35,723 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/A in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:35,723 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/bd4eb98dfbdf461191bb41490a15390b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/713b6b06ad45438cae391cecb2f2d090, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0053a8924cee46a8815c46a772e78752] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=92.2 K 2024-11-27T21:14:35,723 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:35,723 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/bd4eb98dfbdf461191bb41490a15390b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/713b6b06ad45438cae391cecb2f2d090, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0053a8924cee46a8815c46a772e78752] 2024-11-27T21:14:35,724 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:35,724 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/B is initiating minor compaction (all files) 2024-11-27T21:14:35,724 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/B in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:35,724 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/ace608040c564b1a9febaaa7728434f1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/77c0fc26a65745d9ba7ea6441336b5b2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0043a8c5a037476f90046cba488e5863] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=34.3 K 2024-11-27T21:14:35,724 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd4eb98dfbdf461191bb41490a15390b, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732742072541 2024-11-27T21:14:35,724 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting ace608040c564b1a9febaaa7728434f1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732742072541 2024-11-27T21:14:35,725 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 713b6b06ad45438cae391cecb2f2d090, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732742073409 2024-11-27T21:14:35,725 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 77c0fc26a65745d9ba7ea6441336b5b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732742073409 2024-11-27T21:14:35,726 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0053a8924cee46a8815c46a772e78752, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732742074127 2024-11-27T21:14:35,730 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 0043a8c5a037476f90046cba488e5863, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732742074131 2024-11-27T21:14:35,738 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:35,746 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#B#compaction#178 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:35,747 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/a08a604d55f642468a31122ea74b16ac is 50, key is test_row_0/B:col10/1732742074131/Put/seqid=0 2024-11-27T21:14:35,761 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127361787bda3994fdf9d243d5177846431_6d7b62f727fc76cea63daea76cc211d5 store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:35,763 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127361787bda3994fdf9d243d5177846431_6d7b62f727fc76cea63daea76cc211d5, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:35,763 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127361787bda3994fdf9d243d5177846431_6d7b62f727fc76cea63daea76cc211d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:35,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742039_1215 (size=13051) 2024-11-27T21:14:35,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T21:14:35,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:35,771 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-27T21:14:35,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:35,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742040_1216 (size=4469) 2024-11-27T21:14:35,790 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#A#compaction#177 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:35,791 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/b64739d03a70436a8913fddfc9d2e150 is 175, key is test_row_0/A:col10/1732742074131/Put/seqid=0 2024-11-27T21:14:35,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277f2838467bdb4788be80d7e0226924b0_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742074913/Put/seqid=0 2024-11-27T21:14:35,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742041_1217 (size=32112) 2024-11-27T21:14:35,810 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/b64739d03a70436a8913fddfc9d2e150 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/b64739d03a70436a8913fddfc9d2e150 2024-11-27T21:14:35,819 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/A of 6d7b62f727fc76cea63daea76cc211d5 into b64739d03a70436a8913fddfc9d2e150(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:35,819 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:35,819 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/A, priority=13, startTime=1732742075721; duration=0sec 2024-11-27T21:14:35,820 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:35,820 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:A 2024-11-27T21:14:35,820 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:35,821 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:35,822 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/C is initiating minor compaction (all files) 2024-11-27T21:14:35,822 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/C in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:35,822 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/bf1444b25d07456b953ea9c28f284c2f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e75e4ba026fc488e85da2737b2938a67, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/53d8ab0f7992498b8d606d077534bed4] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=34.3 K 2024-11-27T21:14:35,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742042_1218 (size=12454) 2024-11-27T21:14:35,823 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf1444b25d07456b953ea9c28f284c2f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732742072541 2024-11-27T21:14:35,823 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting e75e4ba026fc488e85da2737b2938a67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732742073409 2024-11-27T21:14:35,824 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53d8ab0f7992498b8d606d077534bed4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732742074131 2024-11-27T21:14:35,858 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#C#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:35,859 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/e48b7971ae834171ab3ca422992a711e is 50, key is test_row_0/C:col10/1732742074131/Put/seqid=0 2024-11-27T21:14:35,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742043_1219 (size=13051) 2024-11-27T21:14:35,878 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/e48b7971ae834171ab3ca422992a711e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e48b7971ae834171ab3ca422992a711e 2024-11-27T21:14:35,885 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/C of 6d7b62f727fc76cea63daea76cc211d5 into e48b7971ae834171ab3ca422992a711e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:35,885 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:35,885 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/C, priority=13, startTime=1732742075722; duration=0sec 2024-11-27T21:14:35,885 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:35,885 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:C 2024-11-27T21:14:36,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:36,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742136106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T21:14:36,175 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/a08a604d55f642468a31122ea74b16ac as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/a08a604d55f642468a31122ea74b16ac 2024-11-27T21:14:36,182 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/B of 6d7b62f727fc76cea63daea76cc211d5 into a08a604d55f642468a31122ea74b16ac(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:36,182 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:36,182 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/B, priority=13, startTime=1732742075722; duration=0sec 2024-11-27T21:14:36,182 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:36,182 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:B 2024-11-27T21:14:36,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742136219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:36,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:36,228 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277f2838467bdb4788be80d7e0226924b0_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277f2838467bdb4788be80d7e0226924b0_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:36,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/c9989adf4c3a4331a9e379586d0f8c25, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:36,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/c9989adf4c3a4331a9e379586d0f8c25 is 175, key is test_row_0/A:col10/1732742074913/Put/seqid=0 2024-11-27T21:14:36,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742044_1220 (size=31255) 2024-11-27T21:14:36,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742136429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:36,639 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=429, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/c9989adf4c3a4331a9e379586d0f8c25 2024-11-27T21:14:36,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/7bae39bde18248669a3665c55815479b is 50, key is test_row_0/B:col10/1732742074913/Put/seqid=0 2024-11-27T21:14:36,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742045_1221 (size=12301) 2024-11-27T21:14:36,661 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/7bae39bde18248669a3665c55815479b 2024-11-27T21:14:36,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/6b22d3e00ad74c2a94b07d71f0b08d6b is 50, key is test_row_0/C:col10/1732742074913/Put/seqid=0 2024-11-27T21:14:36,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742046_1222 (size=12301) 2024-11-27T21:14:36,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742136738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:37,082 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/6b22d3e00ad74c2a94b07d71f0b08d6b 2024-11-27T21:14:37,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/c9989adf4c3a4331a9e379586d0f8c25 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/c9989adf4c3a4331a9e379586d0f8c25 2024-11-27T21:14:37,097 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/c9989adf4c3a4331a9e379586d0f8c25, entries=150, sequenceid=429, filesize=30.5 K 2024-11-27T21:14:37,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/7bae39bde18248669a3665c55815479b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/7bae39bde18248669a3665c55815479b 2024-11-27T21:14:37,104 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/7bae39bde18248669a3665c55815479b, entries=150, sequenceid=429, filesize=12.0 K 2024-11-27T21:14:37,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/6b22d3e00ad74c2a94b07d71f0b08d6b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/6b22d3e00ad74c2a94b07d71f0b08d6b 2024-11-27T21:14:37,113 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/6b22d3e00ad74c2a94b07d71f0b08d6b, entries=150, sequenceid=429, filesize=12.0 K 2024-11-27T21:14:37,114 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6d7b62f727fc76cea63daea76cc211d5 in 1343ms, sequenceid=429, compaction requested=false 2024-11-27T21:14:37,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:37,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:37,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-27T21:14:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-27T21:14:37,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-27T21:14:37,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1210 sec 2024-11-27T21:14:37,132 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.1300 sec 2024-11-27T21:14:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T21:14:37,150 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:37,151 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-27T21:14:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T21:14:37,154 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:37,155 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:37,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:37,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-27T21:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T21:14:37,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d4d9e7d749ca47de931a69162b9c917a_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742076087/Put/seqid=0 2024-11-27T21:14:37,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742047_1223 (size=14994) 2024-11-27T21:14:37,306 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:37,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T21:14:37,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:37,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:37,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:37,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:37,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:37,320 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d4d9e7d749ca47de931a69162b9c917a_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d4d9e7d749ca47de931a69162b9c917a_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:37,323 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/0882a5e0efa1469dba512f40b8e4c190, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:37,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/0882a5e0efa1469dba512f40b8e4c190 is 175, key is test_row_0/A:col10/1732742076087/Put/seqid=0 2024-11-27T21:14:37,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742048_1224 (size=39949) 2024-11-27T21:14:37,336 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=445, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/0882a5e0efa1469dba512f40b8e4c190 2024-11-27T21:14:37,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/c3ce0b17f2a245e782b0d8c232c24bb2 is 50, key is test_row_0/B:col10/1732742076087/Put/seqid=0 2024-11-27T21:14:37,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742049_1225 (size=12301) 2024-11-27T21:14:37,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/c3ce0b17f2a245e782b0d8c232c24bb2 2024-11-27T21:14:37,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/e1f1f5abae3143f585bde73780c115be is 50, key is test_row_0/C:col10/1732742076087/Put/seqid=0 2024-11-27T21:14:37,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742050_1226 (size=12301) 2024-11-27T21:14:37,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742137430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:37,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:37,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:37,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:37,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/e1f1f5abae3143f585bde73780c115be 2024-11-27T21:14:37,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/0882a5e0efa1469dba512f40b8e4c190 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0882a5e0efa1469dba512f40b8e4c190 2024-11-27T21:14:37,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0882a5e0efa1469dba512f40b8e4c190, entries=200, sequenceid=445, filesize=39.0 K 2024-11-27T21:14:37,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/c3ce0b17f2a245e782b0d8c232c24bb2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/c3ce0b17f2a245e782b0d8c232c24bb2 2024-11-27T21:14:37,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/c3ce0b17f2a245e782b0d8c232c24bb2, entries=150, sequenceid=445, filesize=12.0 K 2024-11-27T21:14:37,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/e1f1f5abae3143f585bde73780c115be as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e1f1f5abae3143f585bde73780c115be 2024-11-27T21:14:37,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e1f1f5abae3143f585bde73780c115be, entries=150, sequenceid=445, filesize=12.0 K 2024-11-27T21:14:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T21:14:37,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6d7b62f727fc76cea63daea76cc211d5 in 217ms, sequenceid=445, compaction requested=true 2024-11-27T21:14:37,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:37,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:14:37,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:37,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:14:37,470 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:37,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:37,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:14:37,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T21:14:37,470 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:37,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T21:14:37,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:37,473 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T21:14:37,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:37,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:37,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:37,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:37,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:37,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:37,477 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:37,477 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/B is initiating minor compaction (all files) 2024-11-27T21:14:37,477 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/B in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:37,477 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/a08a604d55f642468a31122ea74b16ac, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/7bae39bde18248669a3665c55815479b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/c3ce0b17f2a245e782b0d8c232c24bb2] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=36.8 K 2024-11-27T21:14:37,478 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103316 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:37,478 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/A is initiating minor compaction (all files) 2024-11-27T21:14:37,478 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/A in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:37,478 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/b64739d03a70436a8913fddfc9d2e150, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/c9989adf4c3a4331a9e379586d0f8c25, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0882a5e0efa1469dba512f40b8e4c190] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=100.9 K 2024-11-27T21:14:37,478 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:37,478 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/b64739d03a70436a8913fddfc9d2e150, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/c9989adf4c3a4331a9e379586d0f8c25, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0882a5e0efa1469dba512f40b8e4c190] 2024-11-27T21:14:37,478 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting a08a604d55f642468a31122ea74b16ac, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732742073409 2024-11-27T21:14:37,479 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting b64739d03a70436a8913fddfc9d2e150, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732742073409 2024-11-27T21:14:37,480 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9989adf4c3a4331a9e379586d0f8c25, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732742074882 2024-11-27T21:14:37,480 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bae39bde18248669a3665c55815479b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732742074882 2024-11-27T21:14:37,482 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting c3ce0b17f2a245e782b0d8c232c24bb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732742076087 2024-11-27T21:14:37,482 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0882a5e0efa1469dba512f40b8e4c190, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732742076087 2024-11-27T21:14:37,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d854d237dffb46499e8428c419abdcc1_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742077427/Put/seqid=0 2024-11-27T21:14:37,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742051_1227 (size=12454) 2024-11-27T21:14:37,511 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:37,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:37,514 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#B#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:37,515 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/dd549ea5a6ee4dbd9f6921c62e759013 is 50, key is test_row_0/B:col10/1732742076087/Put/seqid=0 2024-11-27T21:14:37,523 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d854d237dffb46499e8428c419abdcc1_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d854d237dffb46499e8428c419abdcc1_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:37,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/7d09a78f9ee7499394337020656c4221, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:37,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/7d09a78f9ee7499394337020656c4221 is 175, key is test_row_0/A:col10/1732742077427/Put/seqid=0 2024-11-27T21:14:37,532 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127e5dbc5401ee448579ec3b5918e30f7c5_6d7b62f727fc76cea63daea76cc211d5 store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:37,533 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127e5dbc5401ee448579ec3b5918e30f7c5_6d7b62f727fc76cea63daea76cc211d5, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:37,534 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e5dbc5401ee448579ec3b5918e30f7c5_6d7b62f727fc76cea63daea76cc211d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:37,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:37,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742052_1228 (size=13153) 2024-11-27T21:14:37,593 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/dd549ea5a6ee4dbd9f6921c62e759013 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/dd549ea5a6ee4dbd9f6921c62e759013 2024-11-27T21:14:37,605 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/B of 6d7b62f727fc76cea63daea76cc211d5 into dd549ea5a6ee4dbd9f6921c62e759013(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:37,605 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:37,605 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/B, priority=13, startTime=1732742077470; duration=0sec 2024-11-27T21:14:37,605 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:37,605 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:B 2024-11-27T21:14:37,605 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:37,606 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:37,606 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/C is initiating minor compaction (all files) 2024-11-27T21:14:37,606 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/C in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:37,607 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e48b7971ae834171ab3ca422992a711e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/6b22d3e00ad74c2a94b07d71f0b08d6b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e1f1f5abae3143f585bde73780c115be] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=36.8 K 2024-11-27T21:14:37,607 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting e48b7971ae834171ab3ca422992a711e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732742073409 2024-11-27T21:14:37,608 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b22d3e00ad74c2a94b07d71f0b08d6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732742074882 2024-11-27T21:14:37,608 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting e1f1f5abae3143f585bde73780c115be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732742076087 2024-11-27T21:14:37,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742053_1229 (size=31255) 2024-11-27T21:14:37,614 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=464, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/7d09a78f9ee7499394337020656c4221 2024-11-27T21:14:37,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742054_1230 (size=4469) 2024-11-27T21:14:37,647 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#C#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:37,648 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/b4ee5bd76c764f47b8f3afc7b7854902 is 50, key is test_row_0/C:col10/1732742076087/Put/seqid=0 2024-11-27T21:14:37,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/bb77ac18dda5442cbb534b0f29646896 is 50, key is test_row_0/B:col10/1732742077427/Put/seqid=0 2024-11-27T21:14:37,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742137657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:37,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742056_1232 (size=12301) 2024-11-27T21:14:37,725 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/bb77ac18dda5442cbb534b0f29646896 2024-11-27T21:14:37,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742055_1231 (size=13153) 2024-11-27T21:14:37,747 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/b4ee5bd76c764f47b8f3afc7b7854902 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/b4ee5bd76c764f47b8f3afc7b7854902 2024-11-27T21:14:37,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/bb34cc1426574615a0b16dc359ac3507 is 50, key is test_row_0/C:col10/1732742077427/Put/seqid=0 2024-11-27T21:14:37,754 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/C of 6d7b62f727fc76cea63daea76cc211d5 into b4ee5bd76c764f47b8f3afc7b7854902(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:37,754 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:37,754 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/C, priority=13, startTime=1732742077470; duration=0sec 2024-11-27T21:14:37,754 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:37,754 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:C 2024-11-27T21:14:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T21:14:37,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742137781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:37,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:37,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742057_1233 (size=12301) 2024-11-27T21:14:37,795 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/bb34cc1426574615a0b16dc359ac3507 2024-11-27T21:14:37,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/7d09a78f9ee7499394337020656c4221 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/7d09a78f9ee7499394337020656c4221 2024-11-27T21:14:37,812 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/7d09a78f9ee7499394337020656c4221, entries=150, sequenceid=464, filesize=30.5 K 2024-11-27T21:14:37,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/bb77ac18dda5442cbb534b0f29646896 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/bb77ac18dda5442cbb534b0f29646896 2024-11-27T21:14:37,821 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/bb77ac18dda5442cbb534b0f29646896, entries=150, sequenceid=464, filesize=12.0 K 2024-11-27T21:14:37,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/bb34cc1426574615a0b16dc359ac3507 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/bb34cc1426574615a0b16dc359ac3507 2024-11-27T21:14:37,829 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/bb34cc1426574615a0b16dc359ac3507, entries=150, sequenceid=464, filesize=12.0 K 2024-11-27T21:14:37,830 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6d7b62f727fc76cea63daea76cc211d5 in 357ms, sequenceid=464, compaction requested=false 2024-11-27T21:14:37,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:37,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:37,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-27T21:14:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-27T21:14:37,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-27T21:14:37,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 677 msec 2024-11-27T21:14:37,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 685 msec 2024-11-27T21:14:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:37,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T21:14:37,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:37,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:37,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:37,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:37,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:37,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:38,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112770beb500d04849bf935aedef4df23ac1_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742077993/Put/seqid=0 2024-11-27T21:14:38,037 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#A#compaction#187 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:38,039 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/64085972da744bdd9901a74eeeb3ffb5 is 175, key is test_row_0/A:col10/1732742076087/Put/seqid=0 2024-11-27T21:14:38,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742058_1234 (size=14994) 2024-11-27T21:14:38,058 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:38,069 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112770beb500d04849bf935aedef4df23ac1_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112770beb500d04849bf935aedef4df23ac1_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:38,070 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/6f56e61ede40486685c945731467fe1c, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:38,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/6f56e61ede40486685c945731467fe1c is 175, key is test_row_0/A:col10/1732742077993/Put/seqid=0 2024-11-27T21:14:38,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742059_1235 (size=32107) 2024-11-27T21:14:38,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742060_1236 (size=39949) 2024-11-27T21:14:38,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742138109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:38,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:38,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742138219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T21:14:38,289 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:38,290 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:38,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-27T21:14:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T21:14:38,293 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:38,294 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:38,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T21:14:38,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742138429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:38,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:38,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:14:38,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:38,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:38,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:38,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:38,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:38,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60628 deadline: 1732742138480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:38,484 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in 6d7b62f727fc76cea63daea76cc211d5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:38,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60630 deadline: 1732742138500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:38,501 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=480, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/6f56e61ede40486685c945731467fe1c 2024-11-27T21:14:38,501 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_0' in 6d7b62f727fc76cea63daea76cc211d5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:38,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:38,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60624 deadline: 1732742138509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:38,510 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in 6d7b62f727fc76cea63daea76cc211d5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:38,514 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/64085972da744bdd9901a74eeeb3ffb5 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/64085972da744bdd9901a74eeeb3ffb5 2024-11-27T21:14:38,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60618 deadline: 1732742138519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:38,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/95ef43a2e406442780bbbd9a87987627 is 50, key is test_row_0/B:col10/1732742077993/Put/seqid=0 2024-11-27T21:14:38,525 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in 6d7b62f727fc76cea63daea76cc211d5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:38,526 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/A of 6d7b62f727fc76cea63daea76cc211d5 into 64085972da744bdd9901a74eeeb3ffb5(size=31.4 K), total size for store is 61.9 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-27T21:14:38,526 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:38,527 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/A, priority=13, startTime=1732742077470; duration=1sec 2024-11-27T21:14:38,527 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:38,527 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:A 2024-11-27T21:14:38,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742061_1237 (size=12301) 2024-11-27T21:14:38,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/95ef43a2e406442780bbbd9a87987627 2024-11-27T21:14:38,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/0ce91a6908ef4bd5b24b467ae41a823a is 50, key is test_row_0/C:col10/1732742077993/Put/seqid=0 2024-11-27T21:14:38,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742062_1238 (size=12301) 2024-11-27T21:14:38,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/0ce91a6908ef4bd5b24b467ae41a823a 2024-11-27T21:14:38,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/6f56e61ede40486685c945731467fe1c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/6f56e61ede40486685c945731467fe1c 2024-11-27T21:14:38,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:14:38,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:38,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:38,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:38,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:38,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:38,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/6f56e61ede40486685c945731467fe1c, entries=200, sequenceid=480, filesize=39.0 K 2024-11-27T21:14:38,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/95ef43a2e406442780bbbd9a87987627 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/95ef43a2e406442780bbbd9a87987627 2024-11-27T21:14:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T21:14:38,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/95ef43a2e406442780bbbd9a87987627, entries=150, sequenceid=480, filesize=12.0 K 2024-11-27T21:14:38,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/0ce91a6908ef4bd5b24b467ae41a823a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/0ce91a6908ef4bd5b24b467ae41a823a 2024-11-27T21:14:38,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/0ce91a6908ef4bd5b24b467ae41a823a, entries=150, sequenceid=480, filesize=12.0 K 2024-11-27T21:14:38,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6d7b62f727fc76cea63daea76cc211d5 in 630ms, sequenceid=480, compaction requested=true 2024-11-27T21:14:38,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:38,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:14:38,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:38,625 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:38,625 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:38,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:14:38,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:38,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7b62f727fc76cea63daea76cc211d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:14:38,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:38,626 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:38,626 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/A is initiating minor compaction (all files) 2024-11-27T21:14:38,626 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/A in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:38,627 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/64085972da744bdd9901a74eeeb3ffb5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/7d09a78f9ee7499394337020656c4221, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/6f56e61ede40486685c945731467fe1c] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=100.9 K 2024-11-27T21:14:38,627 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:38,627 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/64085972da744bdd9901a74eeeb3ffb5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/7d09a78f9ee7499394337020656c4221, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/6f56e61ede40486685c945731467fe1c] 2024-11-27T21:14:38,627 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:38,627 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64085972da744bdd9901a74eeeb3ffb5, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732742076087 2024-11-27T21:14:38,627 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/B is initiating minor compaction (all files) 2024-11-27T21:14:38,627 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/B in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:38,627 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/dd549ea5a6ee4dbd9f6921c62e759013, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/bb77ac18dda5442cbb534b0f29646896, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/95ef43a2e406442780bbbd9a87987627] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=36.9 K 2024-11-27T21:14:38,628 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d09a78f9ee7499394337020656c4221, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732742077412 2024-11-27T21:14:38,628 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting dd549ea5a6ee4dbd9f6921c62e759013, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732742076087 2024-11-27T21:14:38,628 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f56e61ede40486685c945731467fe1c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1732742077556 2024-11-27T21:14:38,628 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting bb77ac18dda5442cbb534b0f29646896, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732742077412 2024-11-27T21:14:38,629 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ef43a2e406442780bbbd9a87987627, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1732742077556 2024-11-27T21:14:38,636 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:38,663 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:38,663 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/91eaf82be28643989afed2898557e84e is 50, key is test_row_0/B:col10/1732742077993/Put/seqid=0 2024-11-27T21:14:38,664 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127cd11154bae114c1c85df4b43f336f157_6d7b62f727fc76cea63daea76cc211d5 store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:38,666 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127cd11154bae114c1c85df4b43f336f157_6d7b62f727fc76cea63daea76cc211d5, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:38,666 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127cd11154bae114c1c85df4b43f336f157_6d7b62f727fc76cea63daea76cc211d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:38,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742064_1240 (size=13255) 2024-11-27T21:14:38,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742063_1239 (size=4469) 2024-11-27T21:14:38,680 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#A#compaction#195 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:38,681 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/3e140af565f849ab8ba9370172241f2a is 175, key is test_row_0/A:col10/1732742077993/Put/seqid=0 2024-11-27T21:14:38,692 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/91eaf82be28643989afed2898557e84e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/91eaf82be28643989afed2898557e84e 2024-11-27T21:14:38,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742065_1241 (size=32209) 2024-11-27T21:14:38,699 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/B of 6d7b62f727fc76cea63daea76cc211d5 into 91eaf82be28643989afed2898557e84e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:38,700 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:38,700 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/B, priority=13, startTime=1732742078625; duration=0sec 2024-11-27T21:14:38,700 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:38,700 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:B 2024-11-27T21:14:38,700 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:38,701 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:38,701 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 6d7b62f727fc76cea63daea76cc211d5/C is initiating minor compaction (all files) 2024-11-27T21:14:38,701 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7b62f727fc76cea63daea76cc211d5/C in TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:38,701 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/b4ee5bd76c764f47b8f3afc7b7854902, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/bb34cc1426574615a0b16dc359ac3507, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/0ce91a6908ef4bd5b24b467ae41a823a] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp, totalSize=36.9 K 2024-11-27T21:14:38,703 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting b4ee5bd76c764f47b8f3afc7b7854902, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732742076087 2024-11-27T21:14:38,704 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting bb34cc1426574615a0b16dc359ac3507, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732742077412 2024-11-27T21:14:38,705 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/3e140af565f849ab8ba9370172241f2a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/3e140af565f849ab8ba9370172241f2a 2024-11-27T21:14:38,705 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ce91a6908ef4bd5b24b467ae41a823a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1732742077556 2024-11-27T21:14:38,713 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/A of 6d7b62f727fc76cea63daea76cc211d5 into 3e140af565f849ab8ba9370172241f2a(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:38,713 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:38,714 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/A, priority=13, startTime=1732742078625; duration=0sec 2024-11-27T21:14:38,714 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:38,714 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:A 2024-11-27T21:14:38,722 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7b62f727fc76cea63daea76cc211d5#C#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:38,723 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/c709317ef4f740f0a38307f6bbf06f2a is 50, key is test_row_0/C:col10/1732742077993/Put/seqid=0 2024-11-27T21:14:38,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742066_1242 (size=13255) 2024-11-27T21:14:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:38,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T21:14:38,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:38,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:38,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:38,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:38,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:38,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:38,751 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/c709317ef4f740f0a38307f6bbf06f2a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/c709317ef4f740f0a38307f6bbf06f2a 2024-11-27T21:14:38,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127434e64ace6de4d238c2d892dd13e5902_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742078743/Put/seqid=0 2024-11-27T21:14:38,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:14:38,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:38,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:38,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:38,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:38,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:38,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742067_1243 (size=14994) 2024-11-27T21:14:38,762 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7b62f727fc76cea63daea76cc211d5/C of 6d7b62f727fc76cea63daea76cc211d5 into c709317ef4f740f0a38307f6bbf06f2a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:38,762 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:38,762 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., storeName=6d7b62f727fc76cea63daea76cc211d5/C, priority=13, startTime=1732742078626; duration=0sec 2024-11-27T21:14:38,762 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:38,762 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7b62f727fc76cea63daea76cc211d5:C 2024-11-27T21:14:38,762 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:38,766 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127434e64ace6de4d238c2d892dd13e5902_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127434e64ace6de4d238c2d892dd13e5902_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:38,769 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/6345cc7c8358483eafecce9ce1500e14, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:38,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/6345cc7c8358483eafecce9ce1500e14 is 175, key is test_row_0/A:col10/1732742078743/Put/seqid=0 2024-11-27T21:14:38,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742068_1244 (size=39949) 2024-11-27T21:14:38,785 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=502, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/6345cc7c8358483eafecce9ce1500e14 2024-11-27T21:14:38,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/64a649d0dba344ea91d306858578ec8c is 50, key is test_row_0/B:col10/1732742078743/Put/seqid=0 2024-11-27T21:14:38,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742138810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:38,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:38,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742069_1245 (size=12301) 2024-11-27T21:14:38,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:14:38,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:38,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:38,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:38,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:38,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:38,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:38,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T21:14:38,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:38,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742138919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:38,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:38,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:38,926 INFO [Thread-645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-645. 2024-11-27T21:14:38,926 DEBUG [Thread-645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:38,926 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:38,927 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:38,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:38,931 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-27T21:14:38,931 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:38,931 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:38,932 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:38,932 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:38,932 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-11-27T21:14:38,932 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:38,932 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:38,932 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:38,932 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:38,933 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-11-27T21:14:38,933 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:38,933 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:38,933 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:38,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:39,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:14:39,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:39,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:39,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:39,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:39,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:39,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742139129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:39,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:14:39,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:39,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:39,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:39,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:39,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:39,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/64a649d0dba344ea91d306858578ec8c 2024-11-27T21:14:39,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/e0fb6972c50d42b8a514dcd6a160ab5b is 50, key is test_row_0/C:col10/1732742078743/Put/seqid=0 2024-11-27T21:14:39,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742070_1246 (size=12301) 2024-11-27T21:14:39,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:14:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:39,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T21:14:39,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60626 deadline: 1732742139439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:39,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:39,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:39,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:14:39,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:14:39,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:39,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:39,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:39,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:39,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:39,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/e0fb6972c50d42b8a514dcd6a160ab5b 2024-11-27T21:14:39,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/6345cc7c8358483eafecce9ce1500e14 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/6345cc7c8358483eafecce9ce1500e14 2024-11-27T21:14:39,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/6345cc7c8358483eafecce9ce1500e14, entries=200, sequenceid=502, filesize=39.0 K 2024-11-27T21:14:39,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/64a649d0dba344ea91d306858578ec8c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/64a649d0dba344ea91d306858578ec8c 2024-11-27T21:14:39,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/64a649d0dba344ea91d306858578ec8c, entries=150, sequenceid=502, filesize=12.0 K 2024-11-27T21:14:39,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/e0fb6972c50d42b8a514dcd6a160ab5b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e0fb6972c50d42b8a514dcd6a160ab5b 2024-11-27T21:14:39,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e0fb6972c50d42b8a514dcd6a160ab5b, entries=150, sequenceid=502, filesize=12.0 K 2024-11-27T21:14:39,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6d7b62f727fc76cea63daea76cc211d5 in 920ms, sequenceid=502, compaction requested=false 2024-11-27T21:14:39,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:39,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T21:14:39,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:39,676 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-27T21:14:39,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:39,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:39,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:39,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:39,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:39,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:39,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273e39cec55d96408abbdf1180a2e56da5_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742078774/Put/seqid=0 2024-11-27T21:14:39,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742071_1247 (size=12454) 2024-11-27T21:14:39,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:39,703 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273e39cec55d96408abbdf1180a2e56da5_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273e39cec55d96408abbdf1180a2e56da5_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:39,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/eabf730ba6124047b2e5ca43b4d9a06c, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:39,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/eabf730ba6124047b2e5ca43b4d9a06c is 175, key is test_row_0/A:col10/1732742078774/Put/seqid=0 2024-11-27T21:14:39,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742072_1248 (size=31255) 2024-11-27T21:14:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:39,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. as already flushing 2024-11-27T21:14:39,951 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-11-27T21:14:39,951 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:39,951 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:39,951 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:39,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:40,108 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=516, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/eabf730ba6124047b2e5ca43b4d9a06c 2024-11-27T21:14:40,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/73a2ac3804df420489600826af84c2e8 is 50, key is test_row_0/B:col10/1732742078774/Put/seqid=0 2024-11-27T21:14:40,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742073_1249 (size=12301) 2024-11-27T21:14:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T21:14:40,503 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T21:14:40,521 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/73a2ac3804df420489600826af84c2e8 2024-11-27T21:14:40,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/6a7776bff29b414680fa3c40af0435e4 is 50, key is test_row_0/C:col10/1732742078774/Put/seqid=0 2024-11-27T21:14:40,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742074_1250 (size=12301) 2024-11-27T21:14:40,938 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/6a7776bff29b414680fa3c40af0435e4 2024-11-27T21:14:40,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/eabf730ba6124047b2e5ca43b4d9a06c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eabf730ba6124047b2e5ca43b4d9a06c 2024-11-27T21:14:40,958 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eabf730ba6124047b2e5ca43b4d9a06c, entries=150, sequenceid=516, filesize=30.5 K 2024-11-27T21:14:40,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/73a2ac3804df420489600826af84c2e8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/73a2ac3804df420489600826af84c2e8 2024-11-27T21:14:40,963 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/73a2ac3804df420489600826af84c2e8, entries=150, sequenceid=516, filesize=12.0 K 2024-11-27T21:14:40,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/6a7776bff29b414680fa3c40af0435e4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/6a7776bff29b414680fa3c40af0435e4 2024-11-27T21:14:40,967 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/6a7776bff29b414680fa3c40af0435e4, entries=150, sequenceid=516, filesize=12.0 K 2024-11-27T21:14:40,967 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=6.71 KB/6870 for 6d7b62f727fc76cea63daea76cc211d5 in 1291ms, sequenceid=516, compaction requested=true 2024-11-27T21:14:40,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 6d7b62f727fc76cea63daea76cc211d5: 2024-11-27T21:14:40,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:40,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-27T21:14:40,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-27T21:14:40,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-27T21:14:40,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6750 sec 2024-11-27T21:14:40,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.6810 sec 2024-11-27T21:14:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T21:14:42,450 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:48,521 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-11-27T21:14:48,521 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:48,521 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:48,521 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:48,521 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:48,530 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-11-27T21:14:48,530 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:48,530 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:48,530 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:48,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:48,541 INFO [Thread-642 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-642. 2024-11-27T21:14:48,541 DEBUG [Thread-642 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:48,541 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:48,541 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:48,541 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:48,541 INFO [Thread-640 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-640. 2024-11-27T21:14:48,541 DEBUG [Thread-640 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:14:48,541 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:48,541 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:48,541 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T21:14:48,541 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 190 2024-11-27T21:14:48,541 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-27T21:14:48,541 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-27T21:14:48,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-27T21:14:48,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-27T21:14:48,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T21:14:48,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2569 2024-11-27T21:14:48,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2538 2024-11-27T21:14:48,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T21:14:48,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1102 2024-11-27T21:14:48,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3305 rows 2024-11-27T21:14:48,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1100 2024-11-27T21:14:48,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3295 rows 2024-11-27T21:14:48,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:48,542 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T21:14:48,542 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:14:48,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:48,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:48,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:14:48,543 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T21:14:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T21:14:48,546 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742088546"}]},"ts":"1732742088546"} 2024-11-27T21:14:48,547 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T21:14:48,547 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T21:14:48,548 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:14:48,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d7b62f727fc76cea63daea76cc211d5, UNASSIGN}] 2024-11-27T21:14:48,551 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d7b62f727fc76cea63daea76cc211d5, UNASSIGN 2024-11-27T21:14:48,552 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=6d7b62f727fc76cea63daea76cc211d5, regionState=CLOSING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:14:48,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d7b62f727fc76cea63daea76cc211d5, UNASSIGN because future has completed 2024-11-27T21:14:48,553 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:14:48,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:14:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T21:14:48,705 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:48,706 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:14:48,706 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing 6d7b62f727fc76cea63daea76cc211d5, disabling compactions & flushes 2024-11-27T21:14:48,706 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:48,706 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:48,706 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. after waiting 0 ms 2024-11-27T21:14:48,706 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:48,706 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing 6d7b62f727fc76cea63daea76cc211d5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-27T21:14:48,706 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=A 2024-11-27T21:14:48,706 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:48,706 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=B 2024-11-27T21:14:48,706 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:48,706 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d7b62f727fc76cea63daea76cc211d5, store=C 2024-11-27T21:14:48,706 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:48,716 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112726f88ce9e6da400e90dbe773fa2a3c38_6d7b62f727fc76cea63daea76cc211d5 is 50, key is test_row_0/A:col10/1732742088520/Put/seqid=0 2024-11-27T21:14:48,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742075_1251 (size=9914) 2024-11-27T21:14:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T21:14:49,121 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:49,126 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112726f88ce9e6da400e90dbe773fa2a3c38_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112726f88ce9e6da400e90dbe773fa2a3c38_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:49,127 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/8004f19b78b44c99a2600fec010d3f52, store: [table=TestAcidGuarantees family=A region=6d7b62f727fc76cea63daea76cc211d5] 2024-11-27T21:14:49,128 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/8004f19b78b44c99a2600fec010d3f52 is 175, key is test_row_0/A:col10/1732742088520/Put/seqid=0 2024-11-27T21:14:49,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742076_1252 (size=22561) 2024-11-27T21:14:49,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T21:14:49,532 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=524, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/8004f19b78b44c99a2600fec010d3f52 2024-11-27T21:14:49,541 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/84bec3f15cce449ebb248a4c03784f16 is 50, key is test_row_0/B:col10/1732742088520/Put/seqid=0 2024-11-27T21:14:49,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742077_1253 (size=9857) 2024-11-27T21:14:49,546 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/84bec3f15cce449ebb248a4c03784f16 2024-11-27T21:14:49,552 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/2179769103224bb892dd924b9403ff97 is 50, key is test_row_0/C:col10/1732742088520/Put/seqid=0 2024-11-27T21:14:49,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742078_1254 (size=9857) 2024-11-27T21:14:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T21:14:49,957 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/2179769103224bb892dd924b9403ff97 2024-11-27T21:14:49,962 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/A/8004f19b78b44c99a2600fec010d3f52 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/8004f19b78b44c99a2600fec010d3f52 2024-11-27T21:14:49,966 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/8004f19b78b44c99a2600fec010d3f52, entries=100, sequenceid=524, filesize=22.0 K 2024-11-27T21:14:49,967 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/B/84bec3f15cce449ebb248a4c03784f16 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/84bec3f15cce449ebb248a4c03784f16 2024-11-27T21:14:49,971 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/84bec3f15cce449ebb248a4c03784f16, entries=100, sequenceid=524, filesize=9.6 K 2024-11-27T21:14:49,972 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/.tmp/C/2179769103224bb892dd924b9403ff97 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/2179769103224bb892dd924b9403ff97 2024-11-27T21:14:49,976 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/2179769103224bb892dd924b9403ff97, entries=100, sequenceid=524, filesize=9.6 K 2024-11-27T21:14:49,976 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6d7b62f727fc76cea63daea76cc211d5 in 1270ms, sequenceid=524, compaction requested=true 2024-11-27T21:14:49,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d0ff7155b1254cae92d49aa385085d67, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/483a81bec8b54ad0bfb0302df6833262, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/4484eaad0c684544a6d7b3b51547bd4f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/f4ac37da4d8f4aab97b7ec8caf835cb1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eaa10a5ee58249439495be8b8f95bfa4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/ef51e87c6dd64c01b78cea9dfa0bd610, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/a0491ec01d36478683c07654a91e723d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eeb67821368740d190e7bc3d6d9ca8b0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d79913c07acb44e88644f0ed60b38da6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d18294942b2345cb88ba69cf76541613, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/e5b418d945a54f4eb13a632c8e81dfd5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/27711a7b286a483c8a15b151509b1e77, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/8f60321be366466f88e0743d27c93e51, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/f27c49edfebb4ddea006425e0a592ac9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/9cfcad38864a4a8baa80d76cb0f9f1c1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/4c560bc949534343a2377c8008aa0857, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/a32009503d8f42099bf90597e51158e2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/3b483cd725384ad5899f9b6233f9da25, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/bd4eb98dfbdf461191bb41490a15390b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0adc7986b5c54cc8a1530de98a6160a2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/713b6b06ad45438cae391cecb2f2d090, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/b64739d03a70436a8913fddfc9d2e150, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0053a8924cee46a8815c46a772e78752, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/c9989adf4c3a4331a9e379586d0f8c25, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0882a5e0efa1469dba512f40b8e4c190, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/64085972da744bdd9901a74eeeb3ffb5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/7d09a78f9ee7499394337020656c4221, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/6f56e61ede40486685c945731467fe1c] to archive 2024-11-27T21:14:49,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:14:49,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d0ff7155b1254cae92d49aa385085d67 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d0ff7155b1254cae92d49aa385085d67 2024-11-27T21:14:49,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/483a81bec8b54ad0bfb0302df6833262 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/483a81bec8b54ad0bfb0302df6833262 2024-11-27T21:14:49,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/4484eaad0c684544a6d7b3b51547bd4f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/4484eaad0c684544a6d7b3b51547bd4f 2024-11-27T21:14:49,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/f4ac37da4d8f4aab97b7ec8caf835cb1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/f4ac37da4d8f4aab97b7ec8caf835cb1 2024-11-27T21:14:49,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eaa10a5ee58249439495be8b8f95bfa4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eaa10a5ee58249439495be8b8f95bfa4 2024-11-27T21:14:49,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/ef51e87c6dd64c01b78cea9dfa0bd610 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/ef51e87c6dd64c01b78cea9dfa0bd610 2024-11-27T21:14:49,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/a0491ec01d36478683c07654a91e723d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/a0491ec01d36478683c07654a91e723d 2024-11-27T21:14:49,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eeb67821368740d190e7bc3d6d9ca8b0 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eeb67821368740d190e7bc3d6d9ca8b0 2024-11-27T21:14:49,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d79913c07acb44e88644f0ed60b38da6 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d79913c07acb44e88644f0ed60b38da6 2024-11-27T21:14:49,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d18294942b2345cb88ba69cf76541613 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/d18294942b2345cb88ba69cf76541613 2024-11-27T21:14:49,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/e5b418d945a54f4eb13a632c8e81dfd5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/e5b418d945a54f4eb13a632c8e81dfd5 2024-11-27T21:14:49,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/27711a7b286a483c8a15b151509b1e77 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/27711a7b286a483c8a15b151509b1e77 2024-11-27T21:14:49,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/8f60321be366466f88e0743d27c93e51 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/8f60321be366466f88e0743d27c93e51 2024-11-27T21:14:49,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/f27c49edfebb4ddea006425e0a592ac9 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/f27c49edfebb4ddea006425e0a592ac9 2024-11-27T21:14:49,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/9cfcad38864a4a8baa80d76cb0f9f1c1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/9cfcad38864a4a8baa80d76cb0f9f1c1 2024-11-27T21:14:49,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/4c560bc949534343a2377c8008aa0857 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/4c560bc949534343a2377c8008aa0857 2024-11-27T21:14:49,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/a32009503d8f42099bf90597e51158e2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/a32009503d8f42099bf90597e51158e2 2024-11-27T21:14:49,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/3b483cd725384ad5899f9b6233f9da25 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/3b483cd725384ad5899f9b6233f9da25 2024-11-27T21:14:50,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/bd4eb98dfbdf461191bb41490a15390b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/bd4eb98dfbdf461191bb41490a15390b 2024-11-27T21:14:50,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0adc7986b5c54cc8a1530de98a6160a2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0adc7986b5c54cc8a1530de98a6160a2 2024-11-27T21:14:50,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/713b6b06ad45438cae391cecb2f2d090 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/713b6b06ad45438cae391cecb2f2d090 2024-11-27T21:14:50,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/b64739d03a70436a8913fddfc9d2e150 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/b64739d03a70436a8913fddfc9d2e150 2024-11-27T21:14:50,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0053a8924cee46a8815c46a772e78752 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0053a8924cee46a8815c46a772e78752 2024-11-27T21:14:50,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/c9989adf4c3a4331a9e379586d0f8c25 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/c9989adf4c3a4331a9e379586d0f8c25 2024-11-27T21:14:50,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0882a5e0efa1469dba512f40b8e4c190 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/0882a5e0efa1469dba512f40b8e4c190 2024-11-27T21:14:50,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/64085972da744bdd9901a74eeeb3ffb5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/64085972da744bdd9901a74eeeb3ffb5 2024-11-27T21:14:50,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/7d09a78f9ee7499394337020656c4221 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/7d09a78f9ee7499394337020656c4221 2024-11-27T21:14:50,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/6f56e61ede40486685c945731467fe1c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/6f56e61ede40486685c945731467fe1c 2024-11-27T21:14:50,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/af5c6a0ba8064ba1b0977899b53a160c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/fcfa005c14e74c7981a1b712cd3fe4ea, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/a6d3fe13062146d1ac507224b8c9efb8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/4b3bccaa4a9c4e64b749a9412162b025, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/9f3a2656bfd44eaf89967422c94343c8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/eb8a2fc51b80417bbf31d298701476c4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/399410278c5d466099be2a620d7c3023, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/ebfdf4ce3a37419587fb9f6276327891, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0d28a4eebab949c6a192fab37dbd44c8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0c5927ea01264064916a92ace70966fe, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/e14dc3dd40f749709af500f522046eb7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/360415b3ee38480a916a14d2e22741d6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/5c685578278d47c1a378e5eb1e481d3c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/288b5e938ea6424282df2e118daeb1cb, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/a076b91d35ad4eca81147b60b65e5ad1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/abfd9d8390dd43549d1bfcd3e5a81a8b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/2ddc420d5e024fd694233035df99a2bc, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/b597303c06364a80aec6d80fdd105511, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/ace608040c564b1a9febaaa7728434f1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/f6f4e7ac71424ca59a572cfd48bd94bc, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/77c0fc26a65745d9ba7ea6441336b5b2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/a08a604d55f642468a31122ea74b16ac, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0043a8c5a037476f90046cba488e5863, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/7bae39bde18248669a3665c55815479b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/dd549ea5a6ee4dbd9f6921c62e759013, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/c3ce0b17f2a245e782b0d8c232c24bb2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/bb77ac18dda5442cbb534b0f29646896, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/95ef43a2e406442780bbbd9a87987627] to archive 2024-11-27T21:14:50,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:14:50,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/af5c6a0ba8064ba1b0977899b53a160c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/af5c6a0ba8064ba1b0977899b53a160c 2024-11-27T21:14:50,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/fcfa005c14e74c7981a1b712cd3fe4ea to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/fcfa005c14e74c7981a1b712cd3fe4ea 2024-11-27T21:14:50,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/a6d3fe13062146d1ac507224b8c9efb8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/a6d3fe13062146d1ac507224b8c9efb8 2024-11-27T21:14:50,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/4b3bccaa4a9c4e64b749a9412162b025 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/4b3bccaa4a9c4e64b749a9412162b025 2024-11-27T21:14:50,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/9f3a2656bfd44eaf89967422c94343c8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/9f3a2656bfd44eaf89967422c94343c8 2024-11-27T21:14:50,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/eb8a2fc51b80417bbf31d298701476c4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/eb8a2fc51b80417bbf31d298701476c4 2024-11-27T21:14:50,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/399410278c5d466099be2a620d7c3023 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/399410278c5d466099be2a620d7c3023 2024-11-27T21:14:50,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/ebfdf4ce3a37419587fb9f6276327891 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/ebfdf4ce3a37419587fb9f6276327891 2024-11-27T21:14:50,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0d28a4eebab949c6a192fab37dbd44c8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0d28a4eebab949c6a192fab37dbd44c8 2024-11-27T21:14:50,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0c5927ea01264064916a92ace70966fe to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0c5927ea01264064916a92ace70966fe 2024-11-27T21:14:50,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/e14dc3dd40f749709af500f522046eb7 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/e14dc3dd40f749709af500f522046eb7 2024-11-27T21:14:50,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/360415b3ee38480a916a14d2e22741d6 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/360415b3ee38480a916a14d2e22741d6 2024-11-27T21:14:50,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/5c685578278d47c1a378e5eb1e481d3c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/5c685578278d47c1a378e5eb1e481d3c 2024-11-27T21:14:50,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/288b5e938ea6424282df2e118daeb1cb to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/288b5e938ea6424282df2e118daeb1cb 2024-11-27T21:14:50,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/a076b91d35ad4eca81147b60b65e5ad1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/a076b91d35ad4eca81147b60b65e5ad1 2024-11-27T21:14:50,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/abfd9d8390dd43549d1bfcd3e5a81a8b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/abfd9d8390dd43549d1bfcd3e5a81a8b 2024-11-27T21:14:50,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/2ddc420d5e024fd694233035df99a2bc to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/2ddc420d5e024fd694233035df99a2bc 2024-11-27T21:14:50,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/b597303c06364a80aec6d80fdd105511 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/b597303c06364a80aec6d80fdd105511 2024-11-27T21:14:50,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/ace608040c564b1a9febaaa7728434f1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/ace608040c564b1a9febaaa7728434f1 2024-11-27T21:14:50,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/f6f4e7ac71424ca59a572cfd48bd94bc to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/f6f4e7ac71424ca59a572cfd48bd94bc 2024-11-27T21:14:50,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/77c0fc26a65745d9ba7ea6441336b5b2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/77c0fc26a65745d9ba7ea6441336b5b2 2024-11-27T21:14:50,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/a08a604d55f642468a31122ea74b16ac to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/a08a604d55f642468a31122ea74b16ac 2024-11-27T21:14:50,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0043a8c5a037476f90046cba488e5863 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/0043a8c5a037476f90046cba488e5863 2024-11-27T21:14:50,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/7bae39bde18248669a3665c55815479b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/7bae39bde18248669a3665c55815479b 2024-11-27T21:14:50,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/dd549ea5a6ee4dbd9f6921c62e759013 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/dd549ea5a6ee4dbd9f6921c62e759013 2024-11-27T21:14:50,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/c3ce0b17f2a245e782b0d8c232c24bb2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/c3ce0b17f2a245e782b0d8c232c24bb2 2024-11-27T21:14:50,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/bb77ac18dda5442cbb534b0f29646896 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/bb77ac18dda5442cbb534b0f29646896 2024-11-27T21:14:50,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/95ef43a2e406442780bbbd9a87987627 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/95ef43a2e406442780bbbd9a87987627 2024-11-27T21:14:50,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/c0fb4e691a2545b0af82875f400a38f5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/61a360ea542847d6a04c67a43130886f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/0b88b1902a2248a79cefd87bd1514332, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/41f6e58bbe384bf2a866ab14600c6702, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/1d598808def341098605a26b746245b9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/fc1c98e3915d4e7491f37bc34c231c93, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/583225b06c4949f184129969d2f39235, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/97b5d4bd2f4c4f37beae8c01fde3d632, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/1d4b9693ceb3487487f49552d29565ed, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/6a5484bb55114b5ea79fda6454b38c48, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/2704d02f56cd4845afa1a20f979b1715, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/ebee4620d0c34b80befebc0df8d16886, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/ed7ab10a74f74481b07e4d66861d4f26, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/724545c186a1475face2a5553b13271d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/7f0653e5ed2c4aabb2d2f69fc89a71b3, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/699051159bd349e8a7c2253476cef902, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/3b8091ec43e04a6b99cb465218bf6e5c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/683b38c1775c4b28904bd0d9f2450d1e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/bf1444b25d07456b953ea9c28f284c2f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/3743637b6e1346ffaeb08b2cdf803ea3, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e75e4ba026fc488e85da2737b2938a67, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e48b7971ae834171ab3ca422992a711e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/53d8ab0f7992498b8d606d077534bed4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/6b22d3e00ad74c2a94b07d71f0b08d6b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/b4ee5bd76c764f47b8f3afc7b7854902, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e1f1f5abae3143f585bde73780c115be, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/bb34cc1426574615a0b16dc359ac3507, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/0ce91a6908ef4bd5b24b467ae41a823a] to archive 2024-11-27T21:14:50,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:14:50,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/c0fb4e691a2545b0af82875f400a38f5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/c0fb4e691a2545b0af82875f400a38f5 2024-11-27T21:14:50,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/61a360ea542847d6a04c67a43130886f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/61a360ea542847d6a04c67a43130886f 2024-11-27T21:14:50,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/0b88b1902a2248a79cefd87bd1514332 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/0b88b1902a2248a79cefd87bd1514332 2024-11-27T21:14:50,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/41f6e58bbe384bf2a866ab14600c6702 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/41f6e58bbe384bf2a866ab14600c6702 2024-11-27T21:14:50,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/1d598808def341098605a26b746245b9 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/1d598808def341098605a26b746245b9 2024-11-27T21:14:50,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/fc1c98e3915d4e7491f37bc34c231c93 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/fc1c98e3915d4e7491f37bc34c231c93 2024-11-27T21:14:50,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/583225b06c4949f184129969d2f39235 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/583225b06c4949f184129969d2f39235 2024-11-27T21:14:50,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/97b5d4bd2f4c4f37beae8c01fde3d632 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/97b5d4bd2f4c4f37beae8c01fde3d632 2024-11-27T21:14:50,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/1d4b9693ceb3487487f49552d29565ed to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/1d4b9693ceb3487487f49552d29565ed 2024-11-27T21:14:50,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/6a5484bb55114b5ea79fda6454b38c48 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/6a5484bb55114b5ea79fda6454b38c48 2024-11-27T21:14:50,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/2704d02f56cd4845afa1a20f979b1715 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/2704d02f56cd4845afa1a20f979b1715 2024-11-27T21:14:50,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/ebee4620d0c34b80befebc0df8d16886 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/ebee4620d0c34b80befebc0df8d16886 2024-11-27T21:14:50,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/ed7ab10a74f74481b07e4d66861d4f26 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/ed7ab10a74f74481b07e4d66861d4f26 2024-11-27T21:14:50,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/724545c186a1475face2a5553b13271d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/724545c186a1475face2a5553b13271d 2024-11-27T21:14:50,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/7f0653e5ed2c4aabb2d2f69fc89a71b3 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/7f0653e5ed2c4aabb2d2f69fc89a71b3 2024-11-27T21:14:50,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/699051159bd349e8a7c2253476cef902 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/699051159bd349e8a7c2253476cef902 2024-11-27T21:14:50,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/3b8091ec43e04a6b99cb465218bf6e5c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/3b8091ec43e04a6b99cb465218bf6e5c 2024-11-27T21:14:50,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/683b38c1775c4b28904bd0d9f2450d1e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/683b38c1775c4b28904bd0d9f2450d1e 2024-11-27T21:14:50,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/bf1444b25d07456b953ea9c28f284c2f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/bf1444b25d07456b953ea9c28f284c2f 2024-11-27T21:14:50,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/3743637b6e1346ffaeb08b2cdf803ea3 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/3743637b6e1346ffaeb08b2cdf803ea3 2024-11-27T21:14:50,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e75e4ba026fc488e85da2737b2938a67 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e75e4ba026fc488e85da2737b2938a67 2024-11-27T21:14:50,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e48b7971ae834171ab3ca422992a711e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e48b7971ae834171ab3ca422992a711e 2024-11-27T21:14:50,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/53d8ab0f7992498b8d606d077534bed4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/53d8ab0f7992498b8d606d077534bed4 2024-11-27T21:14:50,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/6b22d3e00ad74c2a94b07d71f0b08d6b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/6b22d3e00ad74c2a94b07d71f0b08d6b 2024-11-27T21:14:50,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/b4ee5bd76c764f47b8f3afc7b7854902 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/b4ee5bd76c764f47b8f3afc7b7854902 2024-11-27T21:14:50,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e1f1f5abae3143f585bde73780c115be to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e1f1f5abae3143f585bde73780c115be 2024-11-27T21:14:50,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/bb34cc1426574615a0b16dc359ac3507 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/bb34cc1426574615a0b16dc359ac3507 2024-11-27T21:14:50,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/0ce91a6908ef4bd5b24b467ae41a823a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/0ce91a6908ef4bd5b24b467ae41a823a 2024-11-27T21:14:50,104 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/recovered.edits/527.seqid, newMaxSeqId=527, maxSeqId=4 2024-11-27T21:14:50,105 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. 2024-11-27T21:14:50,105 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for 6d7b62f727fc76cea63daea76cc211d5: Waiting for close lock at 1732742088706Running coprocessor pre-close hooks at 1732742088706Disabling compacts and flushes for region at 1732742088706Disabling writes for close at 1732742088706Obtaining lock to block concurrent updates at 1732742088706Preparing flush snapshotting stores in 6d7b62f727fc76cea63daea76cc211d5 at 1732742088706Finished memstore snapshotting TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732742088706Flushing stores of TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5. at 1732742088707 (+1 ms)Flushing 6d7b62f727fc76cea63daea76cc211d5/A: creating writer at 1732742088707Flushing mob file 6d7b62f727fc76cea63daea76cc211d5/A: appending metadata at 1732742088716 (+9 ms)Flushing mob file 6d7b62f727fc76cea63daea76cc211d5/A: closing flushed file at 1732742088716Flushing 6d7b62f727fc76cea63daea76cc211d5/A: appending metadata at 1732742089127 (+411 ms)Flushing 6d7b62f727fc76cea63daea76cc211d5/A: closing flushed file at 1732742089127Flushing 6d7b62f727fc76cea63daea76cc211d5/B: creating writer at 1732742089537 (+410 ms)Flushing 6d7b62f727fc76cea63daea76cc211d5/B: appending metadata at 1732742089540 (+3 ms)Flushing 6d7b62f727fc76cea63daea76cc211d5/B: closing flushed file at 1732742089540Flushing 6d7b62f727fc76cea63daea76cc211d5/C: creating writer at 1732742089549 (+9 ms)Flushing 6d7b62f727fc76cea63daea76cc211d5/C: appending metadata at 1732742089552 (+3 ms)Flushing 6d7b62f727fc76cea63daea76cc211d5/C: closing flushed file at 1732742089552Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1889492e: reopening flushed file at 1732742089962 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37d816e4: reopening flushed file at 1732742089966 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a640344: reopening flushed file at 1732742089971 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6d7b62f727fc76cea63daea76cc211d5 in 1270ms, sequenceid=524, compaction requested=true at 1732742089976 (+5 ms)Writing region close event to WAL at 1732742090101 (+125 ms)Running coprocessor post-close hooks at 1732742090105 (+4 ms)Closed at 1732742090105 2024-11-27T21:14:50,107 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed 6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,107 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=6d7b62f727fc76cea63daea76cc211d5, regionState=CLOSED 2024-11-27T21:14:50,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:14:50,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-27T21:14:50,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure 6d7b62f727fc76cea63daea76cc211d5, server=4585c39bed03,37381,1732742022265 in 1.5560 sec 2024-11-27T21:14:50,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-27T21:14:50,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d7b62f727fc76cea63daea76cc211d5, UNASSIGN in 1.5610 sec 2024-11-27T21:14:50,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-27T21:14:50,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5650 sec 2024-11-27T21:14:50,115 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742090115"}]},"ts":"1732742090115"} 2024-11-27T21:14:50,116 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T21:14:50,116 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T21:14:50,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5740 sec 2024-11-27T21:14:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T21:14:50,689 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:50,690 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T21:14:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:50,691 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T21:14:50,692 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:50,695 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,697 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/recovered.edits] 2024-11-27T21:14:50,699 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/3e140af565f849ab8ba9370172241f2a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/3e140af565f849ab8ba9370172241f2a 2024-11-27T21:14:50,700 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/6345cc7c8358483eafecce9ce1500e14 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/6345cc7c8358483eafecce9ce1500e14 2024-11-27T21:14:50,701 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/8004f19b78b44c99a2600fec010d3f52 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/8004f19b78b44c99a2600fec010d3f52 2024-11-27T21:14:50,702 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eabf730ba6124047b2e5ca43b4d9a06c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/A/eabf730ba6124047b2e5ca43b4d9a06c 2024-11-27T21:14:50,704 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/64a649d0dba344ea91d306858578ec8c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/64a649d0dba344ea91d306858578ec8c 2024-11-27T21:14:50,705 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/73a2ac3804df420489600826af84c2e8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/73a2ac3804df420489600826af84c2e8 2024-11-27T21:14:50,706 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/84bec3f15cce449ebb248a4c03784f16 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/84bec3f15cce449ebb248a4c03784f16 2024-11-27T21:14:50,707 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/91eaf82be28643989afed2898557e84e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/B/91eaf82be28643989afed2898557e84e 2024-11-27T21:14:50,709 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/2179769103224bb892dd924b9403ff97 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/2179769103224bb892dd924b9403ff97 2024-11-27T21:14:50,710 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/6a7776bff29b414680fa3c40af0435e4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/6a7776bff29b414680fa3c40af0435e4 2024-11-27T21:14:50,711 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/c709317ef4f740f0a38307f6bbf06f2a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/c709317ef4f740f0a38307f6bbf06f2a 2024-11-27T21:14:50,712 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e0fb6972c50d42b8a514dcd6a160ab5b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/C/e0fb6972c50d42b8a514dcd6a160ab5b 2024-11-27T21:14:50,715 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/recovered.edits/527.seqid to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5/recovered.edits/527.seqid 2024-11-27T21:14:50,715 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,715 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T21:14:50,716 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T21:14:50,717 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T21:14:50,722 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112725f8f0f9ad344d65b4cd867c543df140_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112725f8f0f9ad344d65b4cd867c543df140_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,723 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112726f88ce9e6da400e90dbe773fa2a3c38_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112726f88ce9e6da400e90dbe773fa2a3c38_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,725 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127280b749e7b6b4d1faee1d26f2d022d13_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127280b749e7b6b4d1faee1d26f2d022d13_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,726 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272d3df4fe066b4d5b8dcec055d38bf92c_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272d3df4fe066b4d5b8dcec055d38bf92c_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,727 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127390c92f1954445e68e3729013faf2b26_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127390c92f1954445e68e3729013faf2b26_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,728 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273aa7092ce1894f1bbc671fac2baa8b33_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273aa7092ce1894f1bbc671fac2baa8b33_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,730 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273e39cec55d96408abbdf1180a2e56da5_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273e39cec55d96408abbdf1180a2e56da5_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,731 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127416914196145421292e336054a284228_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127416914196145421292e336054a284228_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,732 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127434e64ace6de4d238c2d892dd13e5902_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127434e64ace6de4d238c2d892dd13e5902_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,733 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127468fa2d086b94adeb2fd9ee680079314_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127468fa2d086b94adeb2fd9ee680079314_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,734 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275693ad8af85c4deb86b55f912c3ae64e_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275693ad8af85c4deb86b55f912c3ae64e_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,735 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275e6107f5f54547cca2f4223e2508dd96_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275e6107f5f54547cca2f4223e2508dd96_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,737 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112770beb500d04849bf935aedef4df23ac1_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112770beb500d04849bf935aedef4df23ac1_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,738 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277f2838467bdb4788be80d7e0226924b0_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277f2838467bdb4788be80d7e0226924b0_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,739 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278aa062e3f9254eae86365782412e07a4_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278aa062e3f9254eae86365782412e07a4_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,740 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a0443d3cc1ac4b38b710530d290a84ec_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a0443d3cc1ac4b38b710530d290a84ec_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,741 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127aedc98a53d1f43e494f6a36c7147f85c_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127aedc98a53d1f43e494f6a36c7147f85c_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,742 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b97745bf5d9848b3874e20717cf9847a_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b97745bf5d9848b3874e20717cf9847a_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,743 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c0cd62f1a129418dba6f3d5e55f6131e_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c0cd62f1a129418dba6f3d5e55f6131e_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,745 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d4d9e7d749ca47de931a69162b9c917a_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d4d9e7d749ca47de931a69162b9c917a_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,746 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d854d237dffb46499e8428c419abdcc1_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d854d237dffb46499e8428c419abdcc1_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,747 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127edbe1cd018ee486e8b9cc9417ca09c93_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127edbe1cd018ee486e8b9cc9417ca09c93_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,748 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ee84e0b14988466d931da105b50eed70_6d7b62f727fc76cea63daea76cc211d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ee84e0b14988466d931da105b50eed70_6d7b62f727fc76cea63daea76cc211d5 2024-11-27T21:14:50,748 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T21:14:50,750 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:50,753 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T21:14:50,755 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T21:14:50,756 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:50,756 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T21:14:50,756 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732742090756"}]},"ts":"9223372036854775807"} 2024-11-27T21:14:50,759 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T21:14:50,759 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6d7b62f727fc76cea63daea76cc211d5, NAME => 'TestAcidGuarantees,,1732742055776.6d7b62f727fc76cea63daea76cc211d5.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T21:14:50,759 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T21:14:50,759 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732742090759"}]},"ts":"9223372036854775807"} 2024-11-27T21:14:50,761 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T21:14:50,761 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:50,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 71 msec 2024-11-27T21:14:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T21:14:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T21:14:50,799 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:50,808 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1719228232_22 at /127.0.0.1:46390 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/cluster_5e7b66de-1091-31c3-1f71-380834b464b9/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/cluster_5e7b66de-1091-31c3-1f71-380834b464b9/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=465 (was 380) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8721 (was 9214) 2024-11-27T21:14:50,817 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=465, ProcessCount=11, AvailableMemoryMB=8720 2024-11-27T21:14:50,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:14:50,819 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:14:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T21:14:50,821 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T21:14:50,821 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:50,821 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-27T21:14:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T21:14:50,821 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T21:14:50,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742079_1255 (size=1061) 2024-11-27T21:14:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T21:14:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T21:14:51,233 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de 2024-11-27T21:14:51,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742080_1256 (size=53) 2024-11-27T21:14:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T21:14:51,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:14:51,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing acbb7f305d2f548aa38b4230ffde5235, disabling compactions & flushes 2024-11-27T21:14:51,640 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:51,641 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:51,641 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. after waiting 0 ms 2024-11-27T21:14:51,641 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:51,641 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:51,641 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for acbb7f305d2f548aa38b4230ffde5235: Waiting for close lock at 1732742091640Disabling compacts and flushes for region at 1732742091640Disabling writes for close at 1732742091641 (+1 ms)Writing region close event to WAL at 1732742091641Closed at 1732742091641 2024-11-27T21:14:51,642 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T21:14:51,642 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732742091642"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732742091642"}]},"ts":"1732742091642"} 2024-11-27T21:14:51,644 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T21:14:51,645 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T21:14:51,645 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742091645"}]},"ts":"1732742091645"} 2024-11-27T21:14:51,646 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T21:14:51,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=acbb7f305d2f548aa38b4230ffde5235, ASSIGN}] 2024-11-27T21:14:51,647 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=acbb7f305d2f548aa38b4230ffde5235, ASSIGN 2024-11-27T21:14:51,648 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=acbb7f305d2f548aa38b4230ffde5235, ASSIGN; state=OFFLINE, location=4585c39bed03,37381,1732742022265; forceNewPlan=false, retain=false 2024-11-27T21:14:51,799 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=acbb7f305d2f548aa38b4230ffde5235, regionState=OPENING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:14:51,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=acbb7f305d2f548aa38b4230ffde5235, ASSIGN because future has completed 2024-11-27T21:14:51,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:14:51,940 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T21:14:51,955 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:51,956 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:14:51,956 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:51,956 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:14:51,956 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:51,956 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:51,957 INFO [StoreOpener-acbb7f305d2f548aa38b4230ffde5235-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:51,958 INFO [StoreOpener-acbb7f305d2f548aa38b4230ffde5235-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:14:51,959 INFO [StoreOpener-acbb7f305d2f548aa38b4230ffde5235-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acbb7f305d2f548aa38b4230ffde5235 columnFamilyName A 2024-11-27T21:14:51,959 DEBUG [StoreOpener-acbb7f305d2f548aa38b4230ffde5235-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T21:14:51,959 INFO [StoreOpener-acbb7f305d2f548aa38b4230ffde5235-1 {}] regionserver.HStore(327): Store=acbb7f305d2f548aa38b4230ffde5235/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:14:51,959 INFO [StoreOpener-acbb7f305d2f548aa38b4230ffde5235-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:51,960 INFO [StoreOpener-acbb7f305d2f548aa38b4230ffde5235-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:14:51,960 INFO [StoreOpener-acbb7f305d2f548aa38b4230ffde5235-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acbb7f305d2f548aa38b4230ffde5235 columnFamilyName B 2024-11-27T21:14:51,960 DEBUG [StoreOpener-acbb7f305d2f548aa38b4230ffde5235-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:51,961 INFO [StoreOpener-acbb7f305d2f548aa38b4230ffde5235-1 {}] regionserver.HStore(327): Store=acbb7f305d2f548aa38b4230ffde5235/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:14:51,961 INFO [StoreOpener-acbb7f305d2f548aa38b4230ffde5235-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:51,961 INFO [StoreOpener-acbb7f305d2f548aa38b4230ffde5235-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:14:51,961 INFO [StoreOpener-acbb7f305d2f548aa38b4230ffde5235-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acbb7f305d2f548aa38b4230ffde5235 columnFamilyName C 2024-11-27T21:14:51,961 DEBUG [StoreOpener-acbb7f305d2f548aa38b4230ffde5235-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:14:51,962 INFO [StoreOpener-acbb7f305d2f548aa38b4230ffde5235-1 {}] regionserver.HStore(327): Store=acbb7f305d2f548aa38b4230ffde5235/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:14:51,962 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:51,962 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:51,962 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:51,963 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:51,963 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:51,963 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:51,964 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:14:51,965 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:51,967 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T21:14:51,967 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened acbb7f305d2f548aa38b4230ffde5235; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64903609, jitterRate=-0.03286086022853851}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:14:51,967 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:51,968 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for acbb7f305d2f548aa38b4230ffde5235: Running coprocessor pre-open hook at 1732742091956Writing region info on filesystem at 1732742091956Initializing all the Stores at 1732742091957 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742091957Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742091957Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742091957Cleaning up temporary data from old regions at 1732742091963 (+6 ms)Running coprocessor post-open hooks at 1732742091967 (+4 ms)Region opened successfully at 1732742091968 (+1 ms) 2024-11-27T21:14:51,968 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., pid=67, masterSystemTime=1732742091952 2024-11-27T21:14:51,970 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:51,970 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:51,970 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=acbb7f305d2f548aa38b4230ffde5235, regionState=OPEN, openSeqNum=2, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:14:51,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:14:51,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-27T21:14:51,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 in 172 msec 2024-11-27T21:14:51,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-27T21:14:51,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=acbb7f305d2f548aa38b4230ffde5235, ASSIGN in 327 msec 2024-11-27T21:14:51,976 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T21:14:51,976 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742091976"}]},"ts":"1732742091976"} 2024-11-27T21:14:51,977 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T21:14:51,978 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T21:14:51,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1590 sec 2024-11-27T21:14:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T21:14:52,969 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:52,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,970 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:52,970 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:52,971 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e409e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:52,973 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:52,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@506d6437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,974 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:52,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:52,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:52,977 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T21:14:52,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 4585c39bed03,38503,1732742021440 2024-11-27T21:14:52,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4add4f9a 2024-11-27T21:14:52,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T21:14:52,979 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T21:14:52,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:52,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:52,980 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@404846de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:52,982 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:52,982 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ddd71c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:52,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:52,983 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a5aea00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:52,984 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:52,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d26c804, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:52,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:52,986 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:52,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:52,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:52,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@602d2e2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:52,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:52,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:52,987 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:52,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7685f180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:52,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:52,988 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79f43bf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:52,990 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b70278, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:52,991 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:52,991 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3e8f41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:52,993 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c93afe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:52,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:52,994 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:52,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:52,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:52,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48243b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:52,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:52,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:52,996 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ff1f53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:52,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:52,997 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e026af1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:52,998 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:52,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f4305f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:52,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:52,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:53,000 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@358beaa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:53,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:53,001 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:53,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302cc140, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:53,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:53,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:53,003 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:53,003 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d756aa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:53,004 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46786970, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:53,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:53,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:14:53,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:14:53,006 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:14:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:14:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:14:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc42702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:14:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:14:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:14:53,007 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:14:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1e6eb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:14:53,008 DEBUG [Thread-1186 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:53,008 DEBUG [Thread-1188 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:53,008 DEBUG [Thread-1192 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:53,008 DEBUG [Thread-1203 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:53,008 DEBUG [Thread-1201 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:53,008 DEBUG [Thread-1199 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:53,009 DEBUG [Thread-1184 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:53,009 DEBUG [Thread-1190 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:53,010 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:53,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:53,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:53,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:53,012 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:53,012 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:53,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:53,012 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:53,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:53,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:14:53,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:53,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:14:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:14:53,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-27T21:14:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:14:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:14:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:14:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:14:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T21:14:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:14:53,017 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:53,019 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:53,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:53,021 DEBUG [Thread-1195 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:53,023 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:53,024 DEBUG [Thread-1197 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:14:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:14:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:14:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:14:53,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:14:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:14:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:53,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:14:53,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:14:53,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:53,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:14:53,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:53,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:14:53,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:53,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/7cb024c46f0341d4a3fbbded7f00af53 is 50, key is test_row_0/A:col10/1732742093020/Put/seqid=0 2024-11-27T21:14:53,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742153090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742153093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742153095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742153095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742153096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742081_1257 (size=12001) 2024-11-27T21:14:53,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/7cb024c46f0341d4a3fbbded7f00af53 2024-11-27T21:14:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T21:14:53,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/098053f7982d47c183d8bc838cc984c0 is 50, key is test_row_0/B:col10/1732742093020/Put/seqid=0 2024-11-27T21:14:53,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742082_1258 (size=12001) 2024-11-27T21:14:53,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/098053f7982d47c183d8bc838cc984c0 2024-11-27T21:14:53,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T21:14:53,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:53,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:53,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:53,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/dd7bceeb9ca442f6b5831d7161d1291f is 50, key is test_row_0/C:col10/1732742093020/Put/seqid=0 2024-11-27T21:14:53,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742083_1259 (size=12001) 2024-11-27T21:14:53,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742153208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742153209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742153209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742153209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742153210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T21:14:53,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:53,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:53,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:53,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T21:14:53,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742153419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742153419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742153419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742153419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742153420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T21:14:53,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:53,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:53,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:53,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/dd7bceeb9ca442f6b5831d7161d1291f 2024-11-27T21:14:53,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/7cb024c46f0341d4a3fbbded7f00af53 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/7cb024c46f0341d4a3fbbded7f00af53 2024-11-27T21:14:53,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/7cb024c46f0341d4a3fbbded7f00af53, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T21:14:53,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/098053f7982d47c183d8bc838cc984c0 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/098053f7982d47c183d8bc838cc984c0 2024-11-27T21:14:53,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/098053f7982d47c183d8bc838cc984c0, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T21:14:53,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/dd7bceeb9ca442f6b5831d7161d1291f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/dd7bceeb9ca442f6b5831d7161d1291f 2024-11-27T21:14:53,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T21:14:53,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:53,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/dd7bceeb9ca442f6b5831d7161d1291f, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T21:14:53,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:53,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:53,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for acbb7f305d2f548aa38b4230ffde5235 in 573ms, sequenceid=13, compaction requested=false 2024-11-27T21:14:53,633 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T21:14:53,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:14:53,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T21:14:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:53,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-27T21:14:53,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:14:53,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:53,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:14:53,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:53,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:14:53,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:53,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/8f676558221a401b8f5648968b594257 is 50, key is test_row_0/A:col10/1732742093732/Put/seqid=0 2024-11-27T21:14:53,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742153740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742153741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742153742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742153744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742153744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742084_1260 (size=14341) 2024-11-27T21:14:53,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/8f676558221a401b8f5648968b594257 2024-11-27T21:14:53,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/55968037277b4f35b2d77df8f9871a75 is 50, key is test_row_0/B:col10/1732742093732/Put/seqid=0 2024-11-27T21:14:53,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742085_1261 (size=12001) 2024-11-27T21:14:53,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T21:14:53,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:53,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:53,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:53,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742153849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742153859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742153859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742153859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742153859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:53,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T21:14:53,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:53,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:53,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:53,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:54,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742154059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742154069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742154069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742154069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742154070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T21:14:54,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:54,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:54,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:54,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:54,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T21:14:54,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/55968037277b4f35b2d77df8f9871a75 2024-11-27T21:14:54,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/74a17ecd1c11459dbb65f9f5d1dadeb6 is 50, key is test_row_0/C:col10/1732742093732/Put/seqid=0 2024-11-27T21:14:54,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742086_1262 (size=12001) 2024-11-27T21:14:54,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/74a17ecd1c11459dbb65f9f5d1dadeb6 2024-11-27T21:14:54,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/8f676558221a401b8f5648968b594257 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8f676558221a401b8f5648968b594257 2024-11-27T21:14:54,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8f676558221a401b8f5648968b594257, entries=200, sequenceid=37, filesize=14.0 K 2024-11-27T21:14:54,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/55968037277b4f35b2d77df8f9871a75 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/55968037277b4f35b2d77df8f9871a75 2024-11-27T21:14:54,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/55968037277b4f35b2d77df8f9871a75, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T21:14:54,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/74a17ecd1c11459dbb65f9f5d1dadeb6 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/74a17ecd1c11459dbb65f9f5d1dadeb6 2024-11-27T21:14:54,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/74a17ecd1c11459dbb65f9f5d1dadeb6, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T21:14:54,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~419.77 KB/429840, currentSize=40.25 KB/41220 for acbb7f305d2f548aa38b4230ffde5235 in 472ms, sequenceid=37, compaction requested=false 2024-11-27T21:14:54,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:14:54,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T21:14:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:54,244 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-27T21:14:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:14:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:14:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:14:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:54,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/de9f29a90458421e99c9e6753a310674 is 50, key is test_row_0/A:col10/1732742093740/Put/seqid=0 2024-11-27T21:14:54,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742087_1263 (size=12001) 2024-11-27T21:14:54,254 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/de9f29a90458421e99c9e6753a310674 2024-11-27T21:14:54,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/fa01dcb60db449479008b906dc50e6ae is 50, key is test_row_0/B:col10/1732742093740/Put/seqid=0 2024-11-27T21:14:54,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742088_1264 (size=12001) 2024-11-27T21:14:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:54,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:54,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742094388, max timestamp=1732742094399Num uniques 859; 2024-11-27T21:14:54,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742094388, max timestamp=1732742094399Num uniques 859; 2024-11-27T21:14:54,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:14:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742154396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742094388, max timestamp=1732742094399Num uniques 859; 2024-11-27T21:14:54,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:14:54,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:14:54,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742154424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742154426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742154427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742154431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742154508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742154539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742154539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742154539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742154539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,668 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/fa01dcb60db449479008b906dc50e6ae 2024-11-27T21:14:54,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/53b4f398bd0740e6aa3ab7b13ed0a3cd is 50, key is test_row_0/C:col10/1732742093740/Put/seqid=0 2024-11-27T21:14:54,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742089_1265 (size=12001) 2024-11-27T21:14:54,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742154719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742154749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742154749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742154749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742154749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:55,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742155029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:55,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742155059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:55,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742155059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742155059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:55,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742155059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:55,082 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/53b4f398bd0740e6aa3ab7b13ed0a3cd 2024-11-27T21:14:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/de9f29a90458421e99c9e6753a310674 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/de9f29a90458421e99c9e6753a310674 2024-11-27T21:14:55,092 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/de9f29a90458421e99c9e6753a310674, entries=150, sequenceid=46, filesize=11.7 K 2024-11-27T21:14:55,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/fa01dcb60db449479008b906dc50e6ae as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/fa01dcb60db449479008b906dc50e6ae 2024-11-27T21:14:55,097 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/fa01dcb60db449479008b906dc50e6ae, entries=150, sequenceid=46, filesize=11.7 K 2024-11-27T21:14:55,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/53b4f398bd0740e6aa3ab7b13ed0a3cd as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/53b4f398bd0740e6aa3ab7b13ed0a3cd 2024-11-27T21:14:55,102 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/53b4f398bd0740e6aa3ab7b13ed0a3cd, entries=150, sequenceid=46, filesize=11.7 K 2024-11-27T21:14:55,103 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for acbb7f305d2f548aa38b4230ffde5235 in 859ms, sequenceid=46, compaction requested=true 2024-11-27T21:14:55,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:14:55,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:55,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-27T21:14:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-27T21:14:55,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-27T21:14:55,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0850 sec 2024-11-27T21:14:55,108 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.0930 sec 2024-11-27T21:14:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T21:14:55,179 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:55,179 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-27T21:14:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T21:14:55,181 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:55,182 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:55,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T21:14:55,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T21:14:55,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:55,335 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-27T21:14:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:14:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:14:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:14:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:55,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/1b2cacea7bec4c30a1dbaa20f316381e is 50, key is test_row_0/A:col10/1732742094425/Put/seqid=0 2024-11-27T21:14:55,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742090_1266 (size=12001) 2024-11-27T21:14:55,350 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/1b2cacea7bec4c30a1dbaa20f316381e 2024-11-27T21:14:55,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/feb7d215e90d4cae8f5431a77a37a01d is 50, key is test_row_0/B:col10/1732742094425/Put/seqid=0 2024-11-27T21:14:55,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742091_1267 (size=12001) 2024-11-27T21:14:55,370 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/feb7d215e90d4cae8f5431a77a37a01d 2024-11-27T21:14:55,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/91cb3b909c504b258d1db94c67b0f467 is 50, key is test_row_0/C:col10/1732742094425/Put/seqid=0 2024-11-27T21:14:55,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742092_1268 (size=12001) 2024-11-27T21:14:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T21:14:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:55,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:55,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742155569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:55,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742155569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:55,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742155570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:55,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742155579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:55,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742155579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:55,628 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T21:14:55,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742155681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:55,787 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/91cb3b909c504b258d1db94c67b0f467 2024-11-27T21:14:55,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/1b2cacea7bec4c30a1dbaa20f316381e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/1b2cacea7bec4c30a1dbaa20f316381e 2024-11-27T21:14:55,797 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/1b2cacea7bec4c30a1dbaa20f316381e, entries=150, sequenceid=82, filesize=11.7 K 2024-11-27T21:14:55,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/feb7d215e90d4cae8f5431a77a37a01d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/feb7d215e90d4cae8f5431a77a37a01d 2024-11-27T21:14:55,802 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/feb7d215e90d4cae8f5431a77a37a01d, entries=150, sequenceid=82, filesize=11.7 K 2024-11-27T21:14:55,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/91cb3b909c504b258d1db94c67b0f467 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/91cb3b909c504b258d1db94c67b0f467 2024-11-27T21:14:55,807 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/91cb3b909c504b258d1db94c67b0f467, entries=150, sequenceid=82, filesize=11.7 K 2024-11-27T21:14:55,808 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for acbb7f305d2f548aa38b4230ffde5235 in 474ms, sequenceid=82, compaction requested=true 2024-11-27T21:14:55,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:14:55,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:55,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-27T21:14:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T21:14:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-27T21:14:55,812 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-27T21:14:55,812 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 628 msec 2024-11-27T21:14:55,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 634 msec 2024-11-27T21:14:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:55,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:14:55,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:14:55,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:55,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:14:55,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:55,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:14:55,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:14:55,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/5e47402b372f4a98a94cfc611ddada6f is 50, key is test_row_0/A:col10/1732742095545/Put/seqid=0 2024-11-27T21:14:55,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742093_1269 (size=14341) 2024-11-27T21:14:55,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742095974, max timestamp=1732742095993Num uniques 859; 2024-11-27T21:14:55,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:14:55,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742095974, max timestamp=1732742095993Num uniques 859; 2024-11-27T21:14:55,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:14:55,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742095974, max timestamp=1732742095993Num uniques 859; 2024-11-27T21:14:55,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:14:56,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742156096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:56,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:56,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:56,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:56,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742156209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:56,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/5e47402b372f4a98a94cfc611ddada6f 2024-11-27T21:14:56,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/3b6fa1e0eae64f3ab1eac90ff6a74e2e is 50, key is test_row_0/B:col10/1732742095545/Put/seqid=0 2024-11-27T21:14:56,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742094_1270 (size=12001) 2024-11-27T21:14:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T21:14:56,319 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:56,319 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-27T21:14:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T21:14:56,322 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:56,323 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:56,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:56,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742156419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:56,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:56,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T21:14:56,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T21:14:56,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:56,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:56,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:56,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:56,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:56,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742156589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:56,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742156589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:56,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742156589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:56,592 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in acbb7f305d2f548aa38b4230ffde5235 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:56,592 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in acbb7f305d2f548aa38b4230ffde5235 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:56,592 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_0' in acbb7f305d2f548aa38b4230ffde5235 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:56,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742156599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:56,604 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in acbb7f305d2f548aa38b4230ffde5235 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:14:56,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:56,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:56,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:56,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T21:14:56,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:56,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:56,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:56,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:56,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T21:14:56,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/3b6fa1e0eae64f3ab1eac90ff6a74e2e 2024-11-27T21:14:56,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/d6f4871f5e374c3ea715b998af2cd40f is 50, key is test_row_0/C:col10/1732742095545/Put/seqid=0 2024-11-27T21:14:56,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742095_1271 (size=12001) 2024-11-27T21:14:56,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742156729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:56,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T21:14:56,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:56,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:56,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:56,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:56,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:56,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T21:14:56,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:56,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:56,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:56,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:56,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T21:14:57,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T21:14:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:57,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:57,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/d6f4871f5e374c3ea715b998af2cd40f 2024-11-27T21:14:57,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/5e47402b372f4a98a94cfc611ddada6f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/5e47402b372f4a98a94cfc611ddada6f 2024-11-27T21:14:57,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/5e47402b372f4a98a94cfc611ddada6f, entries=200, sequenceid=92, filesize=14.0 K 2024-11-27T21:14:57,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/3b6fa1e0eae64f3ab1eac90ff6a74e2e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/3b6fa1e0eae64f3ab1eac90ff6a74e2e 2024-11-27T21:14:57,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/3b6fa1e0eae64f3ab1eac90ff6a74e2e, entries=150, sequenceid=92, filesize=11.7 K 2024-11-27T21:14:57,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/d6f4871f5e374c3ea715b998af2cd40f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/d6f4871f5e374c3ea715b998af2cd40f 2024-11-27T21:14:57,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/d6f4871f5e374c3ea715b998af2cd40f, entries=150, sequenceid=92, filesize=11.7 K 2024-11-27T21:14:57,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for acbb7f305d2f548aa38b4230ffde5235 in 1256ms, sequenceid=92, compaction requested=true 2024-11-27T21:14:57,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:14:57,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:14:57,149 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T21:14:57,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:57,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:14:57,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:57,149 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T21:14:57,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:14:57,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:57,150 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T21:14:57,150 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 64685 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T21:14:57,150 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/B is initiating minor compaction (all files) 2024-11-27T21:14:57,150 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/A is initiating minor compaction (all files) 2024-11-27T21:14:57,150 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/B in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:57,150 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/A in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:57,150 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/098053f7982d47c183d8bc838cc984c0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/55968037277b4f35b2d77df8f9871a75, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/fa01dcb60db449479008b906dc50e6ae, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/feb7d215e90d4cae8f5431a77a37a01d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/3b6fa1e0eae64f3ab1eac90ff6a74e2e] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=58.6 K 2024-11-27T21:14:57,150 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/7cb024c46f0341d4a3fbbded7f00af53, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8f676558221a401b8f5648968b594257, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/de9f29a90458421e99c9e6753a310674, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/1b2cacea7bec4c30a1dbaa20f316381e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/5e47402b372f4a98a94cfc611ddada6f] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=63.2 K 2024-11-27T21:14:57,151 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 098053f7982d47c183d8bc838cc984c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732742093020 2024-11-27T21:14:57,151 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cb024c46f0341d4a3fbbded7f00af53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732742093020 2024-11-27T21:14:57,151 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f676558221a401b8f5648968b594257, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732742093090 2024-11-27T21:14:57,151 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 55968037277b4f35b2d77df8f9871a75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732742093732 2024-11-27T21:14:57,152 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting de9f29a90458421e99c9e6753a310674, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732742093740 2024-11-27T21:14:57,152 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting fa01dcb60db449479008b906dc50e6ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732742093740 2024-11-27T21:14:57,152 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b2cacea7bec4c30a1dbaa20f316381e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732742094424 2024-11-27T21:14:57,152 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting feb7d215e90d4cae8f5431a77a37a01d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732742094424 2024-11-27T21:14:57,152 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e47402b372f4a98a94cfc611ddada6f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732742095545 2024-11-27T21:14:57,153 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b6fa1e0eae64f3ab1eac90ff6a74e2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732742095545 2024-11-27T21:14:57,167 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#B#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:57,168 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/305a0aac471446d890cbd0183738c123 is 50, key is test_row_0/B:col10/1732742095545/Put/seqid=0 2024-11-27T21:14:57,176 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#A#compaction#223 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:57,177 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/708922a58f814f76a90ed89b35b81c4b is 50, key is test_row_0/A:col10/1732742095545/Put/seqid=0 2024-11-27T21:14:57,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742096_1272 (size=12173) 2024-11-27T21:14:57,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742097_1273 (size=12173) 2024-11-27T21:14:57,202 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/708922a58f814f76a90ed89b35b81c4b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/708922a58f814f76a90ed89b35b81c4b 2024-11-27T21:14:57,220 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/A of acbb7f305d2f548aa38b4230ffde5235 into 708922a58f814f76a90ed89b35b81c4b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:57,220 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:14:57,220 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/A, priority=11, startTime=1732742097148; duration=0sec 2024-11-27T21:14:57,220 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:57,220 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:A 2024-11-27T21:14:57,220 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T21:14:57,222 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T21:14:57,222 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/C is initiating minor compaction (all files) 2024-11-27T21:14:57,222 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/C in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:57,222 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/dd7bceeb9ca442f6b5831d7161d1291f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/74a17ecd1c11459dbb65f9f5d1dadeb6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/53b4f398bd0740e6aa3ab7b13ed0a3cd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/91cb3b909c504b258d1db94c67b0f467, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/d6f4871f5e374c3ea715b998af2cd40f] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=58.6 K 2024-11-27T21:14:57,223 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd7bceeb9ca442f6b5831d7161d1291f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732742093020 2024-11-27T21:14:57,223 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74a17ecd1c11459dbb65f9f5d1dadeb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732742093732 2024-11-27T21:14:57,224 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53b4f398bd0740e6aa3ab7b13ed0a3cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732742093740 2024-11-27T21:14:57,225 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91cb3b909c504b258d1db94c67b0f467, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732742094424 2024-11-27T21:14:57,225 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6f4871f5e374c3ea715b998af2cd40f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732742095545 2024-11-27T21:14:57,236 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#C#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:57,236 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/c4a9c4449b21427bbc27dbe4b13d9abe is 50, key is test_row_0/C:col10/1732742095545/Put/seqid=0 2024-11-27T21:14:57,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T21:14:57,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:57,243 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-27T21:14:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:57,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732742096047, max timestamp=1732742097243Num uniques 1018; 2024-11-27T21:14:57,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732742096047, max timestamp=1732742097243Num uniques 1018; 2024-11-27T21:14:57,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-27T21:14:57,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732742096047, max timestamp=1732742097243Num uniques 1018; 2024-11-27T21:14:57,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-27T21:14:57,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-27T21:14:57,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:57,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:14:57,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:57,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:14:57,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:57,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:14:57,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:57,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/3260eefccf4e4bca9fd432a2832909ec is 50, key is test_row_0/A:col10/1732742097243/Put/seqid=0 2024-11-27T21:14:57,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742099_1275 (size=14391) 2024-11-27T21:14:57,263 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/3260eefccf4e4bca9fd432a2832909ec 2024-11-27T21:14:57,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742098_1274 (size=12173) 2024-11-27T21:14:57,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/09601a227931429f9db15695535a6090 is 50, key is test_row_0/B:col10/1732742097243/Put/seqid=0 2024-11-27T21:14:57,274 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/c4a9c4449b21427bbc27dbe4b13d9abe as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c4a9c4449b21427bbc27dbe4b13d9abe 2024-11-27T21:14:57,282 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/C of acbb7f305d2f548aa38b4230ffde5235 into c4a9c4449b21427bbc27dbe4b13d9abe(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:57,282 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:14:57,282 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/C, priority=11, startTime=1732742097149; duration=0sec 2024-11-27T21:14:57,283 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:57,283 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:C 2024-11-27T21:14:57,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742100_1276 (size=12051) 2024-11-27T21:14:57,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742097310, max timestamp=1732742097339Num uniques 859; 2024-11-27T21:14:57,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742097310, max timestamp=1732742097339Num uniques 859; 2024-11-27T21:14:57,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:14:57,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:14:57,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742097310, max timestamp=1732742097339Num uniques 859; 2024-11-27T21:14:57,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:14:57,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732742097386, max timestamp=1732742097450Num uniques 1018; 2024-11-27T21:14:57,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-27T21:14:57,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732742097386, max timestamp=1732742097450Num uniques 1018; 2024-11-27T21:14:57,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732742097386, max timestamp=1732742097450Num uniques 1018; 2024-11-27T21:14:57,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-27T21:14:57,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-27T21:14:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T21:14:57,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732742097510, max timestamp=1732742097540Num uniques 1027; 2024-11-27T21:14:57,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732742097510, max timestamp=1732742097540Num uniques 1027; 2024-11-27T21:14:57,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-27T21:14:57,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-27T21:14:57,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732742097510, max timestamp=1732742097540Num uniques 1027; 2024-11-27T21:14:57,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-27T21:14:57,588 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/305a0aac471446d890cbd0183738c123 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/305a0aac471446d890cbd0183738c123 2024-11-27T21:14:57,592 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/B of acbb7f305d2f548aa38b4230ffde5235 into 305a0aac471446d890cbd0183738c123(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:57,592 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:14:57,593 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/B, priority=11, startTime=1732742097149; duration=0sec 2024-11-27T21:14:57,593 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:57,593 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:B 2024-11-27T21:14:57,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732742097600, max timestamp=1732742097631Num uniques 1036; 2024-11-27T21:14:57,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-27T21:14:57,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732742097600, max timestamp=1732742097631Num uniques 1036; 2024-11-27T21:14:57,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-27T21:14:57,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732742097600, max timestamp=1732742097631Num uniques 1036; 2024-11-27T21:14:57,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-27T21:14:57,692 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/09601a227931429f9db15695535a6090 2024-11-27T21:14:57,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/5df4254c12cd497c809708208eabc012 is 50, key is test_row_0/C:col10/1732742097243/Put/seqid=0 2024-11-27T21:14:57,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742101_1277 (size=12051) 2024-11-27T21:14:57,705 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/5df4254c12cd497c809708208eabc012 2024-11-27T21:14:57,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/3260eefccf4e4bca9fd432a2832909ec as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/3260eefccf4e4bca9fd432a2832909ec 2024-11-27T21:14:57,716 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/3260eefccf4e4bca9fd432a2832909ec, entries=200, sequenceid=131, filesize=14.1 K 2024-11-27T21:14:57,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/09601a227931429f9db15695535a6090 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/09601a227931429f9db15695535a6090 2024-11-27T21:14:57,722 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/09601a227931429f9db15695535a6090, entries=150, sequenceid=131, filesize=11.8 K 2024-11-27T21:14:57,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/5df4254c12cd497c809708208eabc012 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/5df4254c12cd497c809708208eabc012 2024-11-27T21:14:57,729 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/5df4254c12cd497c809708208eabc012, entries=150, sequenceid=131, filesize=11.8 K 2024-11-27T21:14:57,730 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=134.18 KB/137400 for acbb7f305d2f548aa38b4230ffde5235 in 486ms, sequenceid=131, compaction requested=false 2024-11-27T21:14:57,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:14:57,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:57,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-27T21:14:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-27T21:14:57,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-27T21:14:57,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-27T21:14:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:57,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4150 sec 2024-11-27T21:14:57,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732742097702, max timestamp=1732742097732Num uniques 1045; 2024-11-27T21:14:57,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-27T21:14:57,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732742097702, max timestamp=1732742097732Num uniques 1045; 2024-11-27T21:14:57,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-27T21:14:57,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732742097702, max timestamp=1732742097732Num uniques 1045; 2024-11-27T21:14:57,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-27T21:14:57,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=26.84 KB heapSize=62.86 KB 2024-11-27T21:14:57,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:14:57,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:57,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:14:57,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:57,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:14:57,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:57,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/86bcf38b399244aba378c6afd0297ad1 is 50, key is test_row_0/A:col10/1732742097725/Put/seqid=0 2024-11-27T21:14:57,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742102_1278 (size=14541) 2024-11-27T21:14:57,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742097800, max timestamp=1732742097818Num uniques 859; 2024-11-27T21:14:57,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742097800, max timestamp=1732742097818Num uniques 859; 2024-11-27T21:14:57,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:14:57,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742097800, max timestamp=1732742097818Num uniques 859; 2024-11-27T21:14:57,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:14:57,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:14:57,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732742097886, max timestamp=1732742097907Num uniques 1018; 2024-11-27T21:14:57,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-27T21:14:57,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732742097886, max timestamp=1732742097907Num uniques 1018; 2024-11-27T21:14:57,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-27T21:14:57,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732742097886, max timestamp=1732742097907Num uniques 1018; 2024-11-27T21:14:57,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-27T21:14:58,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732742097965, max timestamp=1732742098000Num uniques 1027; 2024-11-27T21:14:58,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-27T21:14:58,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732742097965, max timestamp=1732742098000Num uniques 1027; 2024-11-27T21:14:58,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-27T21:14:58,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732742097965, max timestamp=1732742098000Num uniques 1027; 2024-11-27T21:14:58,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-27T21:14:58,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732742098077, max timestamp=1732742098094Num uniques 1036; 2024-11-27T21:14:58,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-27T21:14:58,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732742098077, max timestamp=1732742098094Num uniques 1036; 2024-11-27T21:14:58,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-27T21:14:58,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732742098077, max timestamp=1732742098094Num uniques 1036; 2024-11-27T21:14:58,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-27T21:14:58,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/86bcf38b399244aba378c6afd0297ad1 2024-11-27T21:14:58,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/51ff4fade3f946e99177b408fc31c362 is 50, key is test_row_0/B:col10/1732742097725/Put/seqid=0 2024-11-27T21:14:58,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742103_1279 (size=12151) 2024-11-27T21:14:58,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732742098172, max timestamp=1732742098196Num uniques 1045; 2024-11-27T21:14:58,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-27T21:14:58,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732742098178, max timestamp=1732742098196Num uniques 1045; 2024-11-27T21:14:58,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-27T21:14:58,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732742098178, max timestamp=1732742098196Num uniques 1045; 2024-11-27T21:14:58,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-27T21:14:58,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732742098239, max timestamp=1732742098291Num uniques 1104; 2024-11-27T21:14:58,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-27T21:14:58,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732742098239, max timestamp=1732742098291Num uniques 1054; 2024-11-27T21:14:58,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-27T21:14:58,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732742098239, max timestamp=1732742098291Num uniques 1054; 2024-11-27T21:14:58,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-27T21:14:58,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732742098352, max timestamp=1732742098387Num uniques 1013; 2024-11-27T21:14:58,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-27T21:14:58,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732742098352, max timestamp=1732742098387Num uniques 1013; 2024-11-27T21:14:58,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-27T21:14:58,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732742098352, max timestamp=1732742098387Num uniques 1013; 2024-11-27T21:14:58,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-27T21:14:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T21:14:58,480 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:14:58,480 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:14:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-27T21:14:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T21:14:58,482 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:14:58,483 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:14:58,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:14:58,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732742098455, max timestamp=1732742098486Num uniques 1022; 2024-11-27T21:14:58,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732742098455, max timestamp=1732742098486Num uniques 1022; 2024-11-27T21:14:58,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-27T21:14:58,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-27T21:14:58,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732742098455, max timestamp=1732742098486Num uniques 1022; 2024-11-27T21:14:58,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-27T21:14:58,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/51ff4fade3f946e99177b408fc31c362 2024-11-27T21:14:58,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/f218bc62967f45beae00d58075d5c962 is 50, key is test_row_0/C:col10/1732742097725/Put/seqid=0 2024-11-27T21:14:58,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742104_1280 (size=12151) 2024-11-27T21:14:58,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732742098533, max timestamp=1732742098582Num uniques 1031; 2024-11-27T21:14:58,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732742098533, max timestamp=1732742098582Num uniques 1031; 2024-11-27T21:14:58,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-27T21:14:58,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-27T21:14:58,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732742098533, max timestamp=1732742098582Num uniques 1031; 2024-11-27T21:14:58,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-27T21:14:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T21:14:58,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1732742098609, max timestamp=1732742098625Num uniques 1040; 2024-11-27T21:14:58,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=13280, suffix heap size=136128,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=290 2024-11-27T21:14:58,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1732742098609, max timestamp=1732742098625Num uniques 1040; 2024-11-27T21:14:58,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=13280, suffix heap size=136128,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=290 2024-11-27T21:14:58,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732742098609, max timestamp=1732742098625Num uniques 1040; 2024-11-27T21:14:58,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-27T21:14:58,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T21:14:58,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:58,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:58,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:58,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:58,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:58,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:58,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742158656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:58,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732742098648, max timestamp=1732742098658Num uniques 1149; 2024-11-27T21:14:58,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=11404, suffix heap size=147464,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=249 2024-11-27T21:14:58,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732742098648, max timestamp=1732742098658Num uniques 1099; 2024-11-27T21:14:58,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-27T21:14:58,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732742098648, max timestamp=1732742098658Num uniques 1149; 2024-11-27T21:14:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:58,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=11404, suffix heap size=147464,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=249 2024-11-27T21:14:58,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732742098661, max timestamp=1732742098695Num uniques 1108; 2024-11-27T21:14:58,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-27T21:14:58,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732742098661, max timestamp=1732742098695Num uniques 1108; 2024-11-27T21:14:58,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-27T21:14:58,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732742098661, max timestamp=1732742098695Num uniques 1108; 2024-11-27T21:14:58,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-27T21:14:58,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1732742098722, max timestamp=1732742098729Num uniques 1017; 2024-11-27T21:14:58,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-27T21:14:58,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1732742098722, max timestamp=1732742098729Num uniques 1017; 2024-11-27T21:14:58,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1732742098722, max timestamp=1732742098729Num uniques 1017; 2024-11-27T21:14:58,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-27T21:14:58,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-27T21:14:58,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T21:14:58,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:58,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:58,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:58,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:58,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:58,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=376, cellSize=17216, totalHeapSize=39344, min timestamp=1732742098767, max timestamp=1732742098792Num uniques 1026; 2024-11-27T21:14:58,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=17216, suffix heap size=134672,new segment heap size=39344 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=376 2024-11-27T21:14:58,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=376, cellSize=17216, totalHeapSize=39344, min timestamp=1732742098767, max timestamp=1732742098792Num uniques 1026; 2024-11-27T21:14:58,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=17216, suffix heap size=134672,new segment heap size=39344 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=376 2024-11-27T21:14:58,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=376, cellSize=17216, totalHeapSize=39344, min timestamp=1732742098767, max timestamp=1732742098792Num uniques 1026; 2024-11-27T21:14:58,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=17216, suffix heap size=134672,new segment heap size=39344 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=376 2024-11-27T21:14:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T21:14:58,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:58,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 338 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742158817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742158817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:58,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:58,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742158818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:58,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:58,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:58,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1732742098816, max timestamp=1732742098823Num uniques 1235; 2024-11-27T21:14:58,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=56560, new segment data size=10760, suffix heap size=156408,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1235, new segment cells count=235 2024-11-27T21:14:58,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1732742098816, max timestamp=1732742098823Num uniques 1235; 2024-11-27T21:14:58,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=56560, new segment data size=10760, suffix heap size=156408,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1235, new segment cells count=235 2024-11-27T21:14:58,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1732742098816, max timestamp=1732742098823Num uniques 1235; 2024-11-27T21:14:58,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=56560, new segment data size=10760, suffix heap size=156408,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1235, new segment cells count=235 2024-11-27T21:14:58,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1732742098861, max timestamp=1732742098872Num uniques 1094; 2024-11-27T21:14:58,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-11-27T21:14:58,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1732742098861, max timestamp=1732742098872Num uniques 1094; 2024-11-27T21:14:58,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-11-27T21:14:58,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1732742098861, max timestamp=1732742098872Num uniques 1094; 2024-11-27T21:14:58,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-11-27T21:14:58,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1732742098896, max timestamp=1732742098923Num uniques 1103; 2024-11-27T21:14:58,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1732742098896, max timestamp=1732742098923Num uniques 1103; 2024-11-27T21:14:58,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=7005, suffix heap size=142680,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=153 2024-11-27T21:14:58,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=7005, suffix heap size=142680,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=153 2024-11-27T21:14:58,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1732742098896, max timestamp=1732742098923Num uniques 1103; 2024-11-27T21:14:58,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=7005, suffix heap size=142680,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=153 2024-11-27T21:14:58,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T21:14:58,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:58,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:58,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:58,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:58,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:58,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1732742098936, max timestamp=1732742098956Num uniques 1012; 2024-11-27T21:14:58,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=9702, suffix heap size=133216,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=212 2024-11-27T21:14:58,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1732742098936, max timestamp=1732742098956Num uniques 1012; 2024-11-27T21:14:58,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=9702, suffix heap size=133216,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=212 2024-11-27T21:14:58,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1732742098936, max timestamp=1732742098956Num uniques 1012; 2024-11-27T21:14:58,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=9702, suffix heap size=133216,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=212 2024-11-27T21:14:58,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/f218bc62967f45beae00d58075d5c962 2024-11-27T21:14:58,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/86bcf38b399244aba378c6afd0297ad1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/86bcf38b399244aba378c6afd0297ad1 2024-11-27T21:14:58,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/86bcf38b399244aba378c6afd0297ad1, entries=200, sequenceid=222, filesize=14.2 K 2024-11-27T21:14:58,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/51ff4fade3f946e99177b408fc31c362 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/51ff4fade3f946e99177b408fc31c362 2024-11-27T21:14:58,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1732742098973, max timestamp=1732742098984Num uniques 1071; 2024-11-27T21:14:58,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=10116, suffix heap size=139352,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=221 2024-11-27T21:14:58,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1732742098973, max timestamp=1732742098984Num uniques 1071; 2024-11-27T21:14:58,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=10116, suffix heap size=139352,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=221 2024-11-27T21:14:58,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1732742098973, max timestamp=1732742098984Num uniques 1071; 2024-11-27T21:14:58,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=10116, suffix heap size=139352,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=221 2024-11-27T21:14:58,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/51ff4fade3f946e99177b408fc31c362, entries=150, sequenceid=222, filesize=11.9 K 2024-11-27T21:14:58,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/f218bc62967f45beae00d58075d5c962 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/f218bc62967f45beae00d58075d5c962 2024-11-27T21:14:58,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/f218bc62967f45beae00d58075d5c962, entries=150, sequenceid=222, filesize=11.9 K 2024-11-27T21:14:58,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~62.81 KB/64320, currentSize=60.38 KB/61830 for acbb7f305d2f548aa38b4230ffde5235 in 1259ms, sequenceid=222, compaction requested=true 2024-11-27T21:14:58,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:14:58,997 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:58,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:14:58,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:14:58,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:58,997 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:58,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:14:58,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:58,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:14:58,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:58,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=67.09 KB heapSize=179.95 KB 2024-11-27T21:14:58,998 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41105 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:58,998 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/A is initiating minor compaction (all files) 2024-11-27T21:14:58,998 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/A in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:58,998 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/708922a58f814f76a90ed89b35b81c4b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/3260eefccf4e4bca9fd432a2832909ec, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/86bcf38b399244aba378c6afd0297ad1] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=40.1 K 2024-11-27T21:14:58,998 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36375 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:58,998 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/B is initiating minor compaction (all files) 2024-11-27T21:14:58,998 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/B in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:58,998 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/305a0aac471446d890cbd0183738c123, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/09601a227931429f9db15695535a6090, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/51ff4fade3f946e99177b408fc31c362] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=35.5 K 2024-11-27T21:14:58,999 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 708922a58f814f76a90ed89b35b81c4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732742095545 2024-11-27T21:14:58,999 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 305a0aac471446d890cbd0183738c123, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732742095545 2024-11-27T21:14:58,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:14:58,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:58,999 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3260eefccf4e4bca9fd432a2832909ec, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732742096047 2024-11-27T21:14:58,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:14:58,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:58,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:14:58,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:14:58,999 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 09601a227931429f9db15695535a6090, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732742096088 2024-11-27T21:14:59,000 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 51ff4fade3f946e99177b408fc31c362, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732742097708 2024-11-27T21:14:59,000 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86bcf38b399244aba378c6afd0297ad1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732742097702 2024-11-27T21:14:59,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/a6f82987713444cea4933392eda4f1fd is 50, key is test_row_0/A:col10/1732742098997/Put/seqid=0 2024-11-27T21:14:59,010 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#A#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:59,010 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#B#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:59,011 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/73bab60764f940428dbdb237246d0f82 is 50, key is test_row_0/A:col10/1732742097725/Put/seqid=0 2024-11-27T21:14:59,011 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/e032da030eca496886b3a0f00febb5ff is 50, key is test_row_0/B:col10/1732742097725/Put/seqid=0 2024-11-27T21:14:59,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742159022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742159023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742159025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742159026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742106_1282 (size=12425) 2024-11-27T21:14:59,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 351 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742159029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742105_1281 (size=12301) 2024-11-27T21:14:59,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742107_1283 (size=12425) 2024-11-27T21:14:59,042 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/e032da030eca496886b3a0f00febb5ff as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/e032da030eca496886b3a0f00febb5ff 2024-11-27T21:14:59,047 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/B of acbb7f305d2f548aa38b4230ffde5235 into e032da030eca496886b3a0f00febb5ff(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:59,047 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:14:59,047 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/B, priority=13, startTime=1732742098997; duration=0sec 2024-11-27T21:14:59,048 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:14:59,048 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:B 2024-11-27T21:14:59,048 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:14:59,049 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36375 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:14:59,049 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/C is initiating minor compaction (all files) 2024-11-27T21:14:59,049 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/C in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:59,049 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c4a9c4449b21427bbc27dbe4b13d9abe, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/5df4254c12cd497c809708208eabc012, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/f218bc62967f45beae00d58075d5c962] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=35.5 K 2024-11-27T21:14:59,050 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting c4a9c4449b21427bbc27dbe4b13d9abe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732742095545 2024-11-27T21:14:59,050 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 5df4254c12cd497c809708208eabc012, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732742096088 2024-11-27T21:14:59,051 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting f218bc62967f45beae00d58075d5c962, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732742097708 2024-11-27T21:14:59,057 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#C#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:14:59,057 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/c1262c8893c74a8aaee53dec437df310 is 50, key is test_row_0/C:col10/1732742097725/Put/seqid=0 2024-11-27T21:14:59,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742108_1284 (size=12425) 2024-11-27T21:14:59,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T21:14:59,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:59,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:59,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:59,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T21:14:59,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742159139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742159139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742159139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742159140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 352 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742159140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T21:14:59,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:59,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:59,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:59,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742159348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742159348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742159349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742159349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 353 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742159349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T21:14:59,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:59,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:59,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:59,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/a6f82987713444cea4933392eda4f1fd 2024-11-27T21:14:59,443 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/73bab60764f940428dbdb237246d0f82 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/73bab60764f940428dbdb237246d0f82 2024-11-27T21:14:59,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/12d2a2b8199a478fbe670c75320e7b83 is 50, key is test_row_0/B:col10/1732742098997/Put/seqid=0 2024-11-27T21:14:59,452 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/A of acbb7f305d2f548aa38b4230ffde5235 into 73bab60764f940428dbdb237246d0f82(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:59,452 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:14:59,452 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/A, priority=13, startTime=1732742098996; duration=0sec 2024-11-27T21:14:59,452 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:59,452 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:A 2024-11-27T21:14:59,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742109_1285 (size=12301) 2024-11-27T21:14:59,471 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/c1262c8893c74a8aaee53dec437df310 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c1262c8893c74a8aaee53dec437df310 2024-11-27T21:14:59,475 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/C of acbb7f305d2f548aa38b4230ffde5235 into c1262c8893c74a8aaee53dec437df310(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:14:59,475 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:14:59,475 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/C, priority=13, startTime=1732742098997; duration=0sec 2024-11-27T21:14:59,476 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:14:59,476 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:C 2024-11-27T21:14:59,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T21:14:59,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:59,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:59,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:59,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T21:14:59,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742159659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742159660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742159660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742159660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:14:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 354 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742159660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:14:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:14:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:14:59,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T21:14:59,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:59,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:59,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:59,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/12d2a2b8199a478fbe670c75320e7b83 2024-11-27T21:14:59,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/626ccbc976154dd58f969d784785c9e1 is 50, key is test_row_0/C:col10/1732742098997/Put/seqid=0 2024-11-27T21:14:59,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742110_1286 (size=12301) 2024-11-27T21:14:59,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T21:14:59,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:59,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:14:59,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:14:59,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:14:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:00,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T21:15:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:00,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:00,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742160169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:00,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:00,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742160169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:00,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T21:15:00,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:00,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:00,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:00,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:00,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:00,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742160179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:00,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742160179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:00,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 355 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742160179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:00,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/626ccbc976154dd58f969d784785c9e1 2024-11-27T21:15:00,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/a6f82987713444cea4933392eda4f1fd as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/a6f82987713444cea4933392eda4f1fd 2024-11-27T21:15:00,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/a6f82987713444cea4933392eda4f1fd, entries=150, sequenceid=558, filesize=12.0 K 2024-11-27T21:15:00,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/12d2a2b8199a478fbe670c75320e7b83 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/12d2a2b8199a478fbe670c75320e7b83 2024-11-27T21:15:00,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/12d2a2b8199a478fbe670c75320e7b83, entries=150, sequenceid=558, filesize=12.0 K 2024-11-27T21:15:00,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/626ccbc976154dd58f969d784785c9e1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/626ccbc976154dd58f969d784785c9e1 2024-11-27T21:15:00,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/626ccbc976154dd58f969d784785c9e1, entries=150, sequenceid=558, filesize=12.0 K 2024-11-27T21:15:00,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~199.83 KB/204624, currentSize=107.34 KB/109920 for acbb7f305d2f548aa38b4230ffde5235 in 1289ms, sequenceid=558, compaction requested=false 2024-11-27T21:15:00,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:00,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T21:15:00,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:00,329 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T21:15:00,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:00,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:00,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:00,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:00,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:00,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:00,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/4b55fc51eeff474582ce6eb3ecaf8469 is 50, key is test_row_0/A:col10/1732742099024/Put/seqid=0 2024-11-27T21:15:00,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742111_1287 (size=12301) 2024-11-27T21:15:00,348 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/4b55fc51eeff474582ce6eb3ecaf8469 2024-11-27T21:15:00,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/e38fdd5f04f9417f91d5df829ac2604b is 50, key is test_row_0/B:col10/1732742099024/Put/seqid=0 2024-11-27T21:15:00,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742112_1288 (size=12301) 2024-11-27T21:15:00,371 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/e38fdd5f04f9417f91d5df829ac2604b 2024-11-27T21:15:00,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/b08e6954aed24141843962458ec30f90 is 50, key is test_row_0/C:col10/1732742099024/Put/seqid=0 2024-11-27T21:15:00,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742113_1289 (size=12301) 2024-11-27T21:15:00,397 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/b08e6954aed24141843962458ec30f90 2024-11-27T21:15:00,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/4b55fc51eeff474582ce6eb3ecaf8469 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/4b55fc51eeff474582ce6eb3ecaf8469 2024-11-27T21:15:00,412 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/4b55fc51eeff474582ce6eb3ecaf8469, entries=150, sequenceid=580, filesize=12.0 K 2024-11-27T21:15:00,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/e38fdd5f04f9417f91d5df829ac2604b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/e38fdd5f04f9417f91d5df829ac2604b 2024-11-27T21:15:00,429 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/e38fdd5f04f9417f91d5df829ac2604b, entries=150, sequenceid=580, filesize=12.0 K 2024-11-27T21:15:00,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/b08e6954aed24141843962458ec30f90 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b08e6954aed24141843962458ec30f90 2024-11-27T21:15:00,434 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b08e6954aed24141843962458ec30f90, entries=150, sequenceid=580, filesize=12.0 K 2024-11-27T21:15:00,436 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for acbb7f305d2f548aa38b4230ffde5235 in 107ms, sequenceid=580, compaction requested=true 2024-11-27T21:15:00,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:00,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:00,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-27T21:15:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-27T21:15:00,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-27T21:15:00,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9560 sec 2024-11-27T21:15:00,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.9620 sec 2024-11-27T21:15:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T21:15:00,629 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:00,630 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:15:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-27T21:15:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T21:15:00,632 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:15:00,633 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:15:00,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:15:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T21:15:00,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T21:15:00,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:00,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:00,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:00,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-27T21:15:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-27T21:15:00,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-27T21:15:00,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-27T21:15:00,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 162 msec 2024-11-27T21:15:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T21:15:00,949 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:00,950 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:15:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-27T21:15:00,952 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:15:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T21:15:00,953 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:15:00,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:15:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T21:15:01,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T21:15:01,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:01,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:01,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:01,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-27T21:15:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-27T21:15:01,115 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-27T21:15:01,115 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 158 msec 2024-11-27T21:15:01,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 165 msec 2024-11-27T21:15:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:01,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:15:01,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:01,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:01,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:01,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:01,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:01,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:01,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/21ff212559fe497fbeae2fceb0e08e72 is 50, key is test_row_0/A:col10/1732742101201/Put/seqid=0 2024-11-27T21:15:01,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742114_1290 (size=12301) 2024-11-27T21:15:01,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/21ff212559fe497fbeae2fceb0e08e72 2024-11-27T21:15:01,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/69738dfa28494aa29ca03969d18f0fef is 50, key is test_row_0/B:col10/1732742101201/Put/seqid=0 2024-11-27T21:15:01,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742115_1291 (size=12301) 2024-11-27T21:15:01,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/69738dfa28494aa29ca03969d18f0fef 2024-11-27T21:15:01,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742161231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742161231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742161233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742161234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742101228, max timestamp=1732742101239Num uniques 859; 2024-11-27T21:15:01,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742101228, max timestamp=1732742101239Num uniques 859; 2024-11-27T21:15:01,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:15:01,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742101228, max timestamp=1732742101239Num uniques 859; 2024-11-27T21:15:01,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:15:01,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:15:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/eeeac25d45c54af281741f0ba9df0ae9 is 50, key is test_row_0/C:col10/1732742101201/Put/seqid=0 2024-11-27T21:15:01,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742116_1292 (size=12301) 2024-11-27T21:15:01,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/eeeac25d45c54af281741f0ba9df0ae9 2024-11-27T21:15:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T21:15:01,281 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:01,282 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:15:01,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/21ff212559fe497fbeae2fceb0e08e72 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/21ff212559fe497fbeae2fceb0e08e72 2024-11-27T21:15:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-27T21:15:01,284 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:15:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T21:15:01,285 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:15:01,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:15:01,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/21ff212559fe497fbeae2fceb0e08e72, entries=150, sequenceid=593, filesize=12.0 K 2024-11-27T21:15:01,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/69738dfa28494aa29ca03969d18f0fef as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/69738dfa28494aa29ca03969d18f0fef 2024-11-27T21:15:01,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/69738dfa28494aa29ca03969d18f0fef, entries=150, sequenceid=593, filesize=12.0 K 2024-11-27T21:15:01,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/eeeac25d45c54af281741f0ba9df0ae9 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/eeeac25d45c54af281741f0ba9df0ae9 2024-11-27T21:15:01,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/eeeac25d45c54af281741f0ba9df0ae9, entries=150, sequenceid=593, filesize=12.0 K 2024-11-27T21:15:01,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=80.51 KB/82440 for acbb7f305d2f548aa38b4230ffde5235 in 103ms, sequenceid=593, compaction requested=true 2024-11-27T21:15:01,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:01,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:15:01,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:01,305 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:15:01,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:15:01,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:01,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:01,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T21:15:01,305 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:15:01,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=87.22 KB heapSize=245.53 KB 2024-11-27T21:15:01,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:01,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:01,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:01,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:01,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:01,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:01,310 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:15:01,310 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/A is initiating minor compaction (all files) 2024-11-27T21:15:01,310 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/A in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:01,310 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/73bab60764f940428dbdb237246d0f82, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/a6f82987713444cea4933392eda4f1fd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/4b55fc51eeff474582ce6eb3ecaf8469, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/21ff212559fe497fbeae2fceb0e08e72] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=48.2 K 2024-11-27T21:15:01,311 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:15:01,311 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/B is initiating minor compaction (all files) 2024-11-27T21:15:01,311 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/B in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:01,311 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/e032da030eca496886b3a0f00febb5ff, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/12d2a2b8199a478fbe670c75320e7b83, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/e38fdd5f04f9417f91d5df829ac2604b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/69738dfa28494aa29ca03969d18f0fef] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=48.2 K 2024-11-27T21:15:01,312 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73bab60764f940428dbdb237246d0f82, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732742097708 2024-11-27T21:15:01,312 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting e032da030eca496886b3a0f00febb5ff, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732742097708 2024-11-27T21:15:01,312 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6f82987713444cea4933392eda4f1fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1732742098991 2024-11-27T21:15:01,313 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 12d2a2b8199a478fbe670c75320e7b83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1732742098991 2024-11-27T21:15:01,318 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting e38fdd5f04f9417f91d5df829ac2604b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1732742099013 2024-11-27T21:15:01,318 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b55fc51eeff474582ce6eb3ecaf8469, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1732742099013 2024-11-27T21:15:01,318 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 69738dfa28494aa29ca03969d18f0fef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1732742101190 2024-11-27T21:15:01,318 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21ff212559fe497fbeae2fceb0e08e72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1732742101190 2024-11-27T21:15:01,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/1669f3a1ef8a473692cb42d0cfeee317 is 50, key is test_row_0/A:col10/1732742101307/Put/seqid=0 2024-11-27T21:15:01,331 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#B#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:01,331 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/2f2bf22489444ec89cdfa717a87c3d50 is 50, key is test_row_0/B:col10/1732742101201/Put/seqid=0 2024-11-27T21:15:01,331 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#A#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:01,334 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/738ec392121a4744aac4aa8927e29462 is 50, key is test_row_0/A:col10/1732742101201/Put/seqid=0 2024-11-27T21:15:01,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742119_1295 (size=12711) 2024-11-27T21:15:01,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742117_1293 (size=14741) 2024-11-27T21:15:01,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/1669f3a1ef8a473692cb42d0cfeee317 2024-11-27T21:15:01,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742118_1294 (size=12711) 2024-11-27T21:15:01,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742161362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 378 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742161368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/88d25563360542a2ac76a3cfbce7d0d2 is 50, key is test_row_0/B:col10/1732742101307/Put/seqid=0 2024-11-27T21:15:01,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742161370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742161371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742161372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T21:15:01,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742120_1296 (size=12301) 2024-11-27T21:15:01,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/88d25563360542a2ac76a3cfbce7d0d2 2024-11-27T21:15:01,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/c80acf6950294716ad03ace62af91d7d is 50, key is test_row_0/C:col10/1732742101307/Put/seqid=0 2024-11-27T21:15:01,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742121_1297 (size=12301) 2024-11-27T21:15:01,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/c80acf6950294716ad03ace62af91d7d 2024-11-27T21:15:01,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/1669f3a1ef8a473692cb42d0cfeee317 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/1669f3a1ef8a473692cb42d0cfeee317 2024-11-27T21:15:01,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T21:15:01,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:01,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:01,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:01,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:01,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:01,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/1669f3a1ef8a473692cb42d0cfeee317, entries=200, sequenceid=623, filesize=14.4 K 2024-11-27T21:15:01,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/88d25563360542a2ac76a3cfbce7d0d2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/88d25563360542a2ac76a3cfbce7d0d2 2024-11-27T21:15:01,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/88d25563360542a2ac76a3cfbce7d0d2, entries=150, sequenceid=623, filesize=12.0 K 2024-11-27T21:15:01,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/c80acf6950294716ad03ace62af91d7d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c80acf6950294716ad03ace62af91d7d 2024-11-27T21:15:01,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c80acf6950294716ad03ace62af91d7d, entries=150, sequenceid=623, filesize=12.0 K 2024-11-27T21:15:01,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~245.48 KB/251376, currentSize=93.93 KB/96180 for acbb7f305d2f548aa38b4230ffde5235 in 151ms, sequenceid=623, compaction requested=true 2024-11-27T21:15:01,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:01,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:A, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:01,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T21:15:01,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:B, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:01,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-27T21:15:01,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:01,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-27T21:15:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:01,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-27T21:15:01,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:01,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:01,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:01,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:01,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:01,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:01,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/6f95e1c1ed63498aaedac7154270a260 is 50, key is test_row_0/A:col10/1732742101369/Put/seqid=0 2024-11-27T21:15:01,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742122_1298 (size=12301) 2024-11-27T21:15:01,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742161540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742161541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742161541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 382 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742161548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742161548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T21:15:01,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:01,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:01,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:01,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:01,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T21:15:01,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742161659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742161659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742161659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 383 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742161659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742161660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T21:15:01,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:01,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:01,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:01,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:01,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:01,757 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/738ec392121a4744aac4aa8927e29462 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/738ec392121a4744aac4aa8927e29462 2024-11-27T21:15:01,765 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/A of acbb7f305d2f548aa38b4230ffde5235 into 738ec392121a4744aac4aa8927e29462(size=12.4 K), total size for store is 26.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:01,765 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:01,765 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/A, priority=12, startTime=1732742101305; duration=0sec 2024-11-27T21:15:01,765 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-27T21:15:01,765 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:A 2024-11-27T21:15:01,765 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:A 2024-11-27T21:15:01,765 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T21:15:01,767 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61629 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T21:15:01,768 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/2f2bf22489444ec89cdfa717a87c3d50 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/2f2bf22489444ec89cdfa717a87c3d50 2024-11-27T21:15:01,768 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/C is initiating minor compaction (all files) 2024-11-27T21:15:01,768 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/C in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:01,768 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c1262c8893c74a8aaee53dec437df310, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/626ccbc976154dd58f969d784785c9e1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b08e6954aed24141843962458ec30f90, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/eeeac25d45c54af281741f0ba9df0ae9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c80acf6950294716ad03ace62af91d7d] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=60.2 K 2024-11-27T21:15:01,769 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1262c8893c74a8aaee53dec437df310, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732742097708 2024-11-27T21:15:01,769 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 626ccbc976154dd58f969d784785c9e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1732742098991 2024-11-27T21:15:01,770 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting b08e6954aed24141843962458ec30f90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1732742099013 2024-11-27T21:15:01,771 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeeac25d45c54af281741f0ba9df0ae9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1732742101190 2024-11-27T21:15:01,771 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting c80acf6950294716ad03ace62af91d7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=623, earliestPutTs=1732742101271 2024-11-27T21:15:01,773 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/B of acbb7f305d2f548aa38b4230ffde5235 into 2f2bf22489444ec89cdfa717a87c3d50(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:01,773 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:01,773 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/B, priority=12, startTime=1732742101305; duration=0sec 2024-11-27T21:15:01,773 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-27T21:15:01,773 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:B 2024-11-27T21:15:01,774 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:B 2024-11-27T21:15:01,774 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-27T21:15:01,774 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T21:15:01,774 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T21:15:01,774 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. because compaction request was cancelled 2024-11-27T21:15:01,774 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:C 2024-11-27T21:15:01,774 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T21:15:01,776 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T21:15:01,776 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T21:15:01,777 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. because compaction request was cancelled 2024-11-27T21:15:01,777 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:B 2024-11-27T21:15:01,777 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T21:15:01,778 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T21:15:01,778 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T21:15:01,778 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. because compaction request was cancelled 2024-11-27T21:15:01,778 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:A 2024-11-27T21:15:01,787 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#C#compaction#249 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:01,788 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/c515bb5eac7847769a689a257b29755a is 50, key is test_row_0/C:col10/1732742101307/Put/seqid=0 2024-11-27T21:15:01,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742123_1299 (size=12745) 2024-11-27T21:15:01,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742161869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742161869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742161869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 384 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742161869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:01,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742161869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:01,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:01,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:01,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T21:15:01,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:01,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:01,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:01,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:01,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T21:15:01,940 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T21:15:01,940 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-27T21:15:01,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=641 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/6f95e1c1ed63498aaedac7154270a260 2024-11-27T21:15:01,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/b9093b2f18804df399736f038b602ac7 is 50, key is test_row_0/B:col10/1732742101369/Put/seqid=0 2024-11-27T21:15:01,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742124_1300 (size=12301) 2024-11-27T21:15:02,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T21:15:02,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:02,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:02,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:02,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:02,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:02,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742162179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:02,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742162179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:02,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 385 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742162179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:02,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742162179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:02,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:02,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:02,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742162179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:02,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:02,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:02,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T21:15:02,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:02,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:02,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:02,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:02,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:02,207 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/c515bb5eac7847769a689a257b29755a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c515bb5eac7847769a689a257b29755a 2024-11-27T21:15:02,212 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/C of acbb7f305d2f548aa38b4230ffde5235 into c515bb5eac7847769a689a257b29755a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:02,212 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:02,212 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/C, priority=11, startTime=1732742101462; duration=0sec 2024-11-27T21:15:02,212 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:02,212 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:C 2024-11-27T21:15:02,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=641 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/b9093b2f18804df399736f038b602ac7 2024-11-27T21:15:02,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T21:15:02,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:02,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:02,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:02,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:02,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:02,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/b5f872fbfa6d4b648cc3c23941028a16 is 50, key is test_row_0/C:col10/1732742101369/Put/seqid=0 2024-11-27T21:15:02,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742125_1301 (size=12301) 2024-11-27T21:15:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T21:15:02,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T21:15:02,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:02,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:02,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:02,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:02,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:02,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:02,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T21:15:02,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:02,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:02,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:02,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:02,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:02,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742162689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:02,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742162689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:02,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742162689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742162689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:02,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 386 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742162699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:02,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:02,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:02,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=641 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/b5f872fbfa6d4b648cc3c23941028a16 2024-11-27T21:15:02,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/6f95e1c1ed63498aaedac7154270a260 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/6f95e1c1ed63498aaedac7154270a260 2024-11-27T21:15:02,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/6f95e1c1ed63498aaedac7154270a260, entries=150, sequenceid=641, filesize=12.0 K 2024-11-27T21:15:02,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/b9093b2f18804df399736f038b602ac7 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/b9093b2f18804df399736f038b602ac7 2024-11-27T21:15:02,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/b9093b2f18804df399736f038b602ac7, entries=150, sequenceid=641, filesize=12.0 K 2024-11-27T21:15:02,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/b5f872fbfa6d4b648cc3c23941028a16 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b5f872fbfa6d4b648cc3c23941028a16 2024-11-27T21:15:02,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b5f872fbfa6d4b648cc3c23941028a16, entries=150, sequenceid=641, filesize=12.0 K 2024-11-27T21:15:02,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for acbb7f305d2f548aa38b4230ffde5235 in 1301ms, sequenceid=641, compaction requested=true 2024-11-27T21:15:02,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:02,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:15:02,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:02,787 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:02,787 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:02,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:15:02,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:02,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:02,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:02,788 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39753 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:02,788 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/A is initiating minor compaction (all files) 2024-11-27T21:15:02,788 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/A in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:02,788 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/738ec392121a4744aac4aa8927e29462, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/1669f3a1ef8a473692cb42d0cfeee317, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/6f95e1c1ed63498aaedac7154270a260] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=38.8 K 2024-11-27T21:15:02,788 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:02,788 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/B is initiating minor compaction (all files) 2024-11-27T21:15:02,788 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/B in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:02,789 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/2f2bf22489444ec89cdfa717a87c3d50, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/88d25563360542a2ac76a3cfbce7d0d2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/b9093b2f18804df399736f038b602ac7] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=36.4 K 2024-11-27T21:15:02,789 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f2bf22489444ec89cdfa717a87c3d50, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1732742101190 2024-11-27T21:15:02,789 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 738ec392121a4744aac4aa8927e29462, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1732742101190 2024-11-27T21:15:02,790 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 88d25563360542a2ac76a3cfbce7d0d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=623, earliestPutTs=1732742101271 2024-11-27T21:15:02,790 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1669f3a1ef8a473692cb42d0cfeee317, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=623, earliestPutTs=1732742101271 2024-11-27T21:15:02,790 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting b9093b2f18804df399736f038b602ac7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=641, earliestPutTs=1732742101367 2024-11-27T21:15:02,790 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f95e1c1ed63498aaedac7154270a260, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=641, earliestPutTs=1732742101367 2024-11-27T21:15:02,800 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#A#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:02,800 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:02,801 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/e9a6047cdb0741d099392d371f88bafb is 50, key is test_row_0/A:col10/1732742101369/Put/seqid=0 2024-11-27T21:15:02,801 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/34821fedb5bf43718dccda40a757da6e is 50, key is test_row_0/B:col10/1732742101369/Put/seqid=0 2024-11-27T21:15:02,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742126_1302 (size=12813) 2024-11-27T21:15:02,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742127_1303 (size=12813) 2024-11-27T21:15:02,816 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/34821fedb5bf43718dccda40a757da6e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/34821fedb5bf43718dccda40a757da6e 2024-11-27T21:15:02,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T21:15:02,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:02,819 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-27T21:15:02,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:02,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:02,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:02,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:02,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:02,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:02,821 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/B of acbb7f305d2f548aa38b4230ffde5235 into 34821fedb5bf43718dccda40a757da6e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:02,821 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:02,821 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/B, priority=13, startTime=1732742102787; duration=0sec 2024-11-27T21:15:02,822 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:02,822 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:B 2024-11-27T21:15:02,822 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T21:15:02,823 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T21:15:02,823 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T21:15:02,823 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. because compaction request was cancelled 2024-11-27T21:15:02,823 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:C 2024-11-27T21:15:02,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/8c3b621a8261421b8f58278b6f813e45 is 50, key is test_row_0/A:col10/1732742101539/Put/seqid=0 2024-11-27T21:15:02,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742128_1304 (size=12301) 2024-11-27T21:15:03,217 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/e9a6047cdb0741d099392d371f88bafb as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/e9a6047cdb0741d099392d371f88bafb 2024-11-27T21:15:03,222 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/A of acbb7f305d2f548aa38b4230ffde5235 into e9a6047cdb0741d099392d371f88bafb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:03,222 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:03,222 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/A, priority=13, startTime=1732742102787; duration=0sec 2024-11-27T21:15:03,222 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:03,222 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:A 2024-11-27T21:15:03,232 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=659 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/8c3b621a8261421b8f58278b6f813e45 2024-11-27T21:15:03,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/6ef55cda8c2e4ab3a4c672b11d32f053 is 50, key is test_row_0/B:col10/1732742101539/Put/seqid=0 2024-11-27T21:15:03,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742129_1305 (size=12301) 2024-11-27T21:15:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T21:15:03,648 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=659 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/6ef55cda8c2e4ab3a4c672b11d32f053 2024-11-27T21:15:03,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/3ca52e60831e4030adbd0ce07c5fcd69 is 50, key is test_row_0/C:col10/1732742101539/Put/seqid=0 2024-11-27T21:15:03,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742130_1306 (size=12301) 2024-11-27T21:15:03,661 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=659 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/3ca52e60831e4030adbd0ce07c5fcd69 2024-11-27T21:15:03,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/8c3b621a8261421b8f58278b6f813e45 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8c3b621a8261421b8f58278b6f813e45 2024-11-27T21:15:03,674 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8c3b621a8261421b8f58278b6f813e45, entries=150, sequenceid=659, filesize=12.0 K 2024-11-27T21:15:03,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/6ef55cda8c2e4ab3a4c672b11d32f053 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/6ef55cda8c2e4ab3a4c672b11d32f053 2024-11-27T21:15:03,679 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/6ef55cda8c2e4ab3a4c672b11d32f053, entries=150, sequenceid=659, filesize=12.0 K 2024-11-27T21:15:03,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/3ca52e60831e4030adbd0ce07c5fcd69 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/3ca52e60831e4030adbd0ce07c5fcd69 2024-11-27T21:15:03,684 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/3ca52e60831e4030adbd0ce07c5fcd69, entries=150, sequenceid=659, filesize=12.0 K 2024-11-27T21:15:03,684 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for acbb7f305d2f548aa38b4230ffde5235 in 865ms, sequenceid=659, compaction requested=true 2024-11-27T21:15:03,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:03,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:03,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-27T21:15:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-27T21:15:03,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-27T21:15:03,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4010 sec 2024-11-27T21:15:03,689 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.4060 sec 2024-11-27T21:15:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:03,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:15:03,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:03,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:03,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:03,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:03,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:03,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:03,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/100585a8335c4e7f8015c337ca0f9d74 is 50, key is test_row_0/A:col10/1732742103711/Put/seqid=0 2024-11-27T21:15:03,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742131_1307 (size=12301) 2024-11-27T21:15:03,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/100585a8335c4e7f8015c337ca0f9d74 2024-11-27T21:15:03,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/532f713802074883b0d2adc1f54c4608 is 50, key is test_row_0/B:col10/1732742103711/Put/seqid=0 2024-11-27T21:15:03,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742132_1308 (size=12301) 2024-11-27T21:15:03,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742103736, max timestamp=1732742103766Num uniques 859; 2024-11-27T21:15:03,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:15:03,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742103736, max timestamp=1732742103766Num uniques 859; 2024-11-27T21:15:03,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:15:03,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742103736, max timestamp=1732742103766Num uniques 859; 2024-11-27T21:15:03,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:15:03,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:03,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 394 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742163817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742163817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:03,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732742103775, max timestamp=1732742103822Num uniques 1018; 2024-11-27T21:15:03,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-27T21:15:03,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732742103775, max timestamp=1732742103822Num uniques 1018; 2024-11-27T21:15:03,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-27T21:15:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:03,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=182, cellSize=8332, totalHeapSize=19168, min timestamp=1732742103775, max timestamp=1732742103822Num uniques 1018; 2024-11-27T21:15:03,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=8332, suffix heap size=133840,new segment heap size=19168 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=182 2024-11-27T21:15:03,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742163866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:03,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:03,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742163866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742163868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:03,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 395 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742163928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:03,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:03,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742163939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:03,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742163979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:03,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742163979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:03,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742163979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:04,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/532f713802074883b0d2adc1f54c4608 2024-11-27T21:15:04,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/6259df03d2a149ceb4166612b336783d is 50, key is test_row_0/C:col10/1732742103711/Put/seqid=0 2024-11-27T21:15:04,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 396 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742164139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:04,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742133_1309 (size=12301) 2024-11-27T21:15:04,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742164149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:04,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742164188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:04,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742164188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:04,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:04,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742164189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:04,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:04,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:04,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 397 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742164449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:04,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742164464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:04,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:04,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:04,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742164499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:04,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:04,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742164499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:04,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:04,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:04,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742164500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:04,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:04,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:04,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/6259df03d2a149ceb4166612b336783d 2024-11-27T21:15:04,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/100585a8335c4e7f8015c337ca0f9d74 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/100585a8335c4e7f8015c337ca0f9d74 2024-11-27T21:15:04,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/100585a8335c4e7f8015c337ca0f9d74, entries=150, sequenceid=671, filesize=12.0 K 2024-11-27T21:15:04,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/532f713802074883b0d2adc1f54c4608 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/532f713802074883b0d2adc1f54c4608 2024-11-27T21:15:04,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/532f713802074883b0d2adc1f54c4608, entries=150, sequenceid=671, filesize=12.0 K 2024-11-27T21:15:04,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/6259df03d2a149ceb4166612b336783d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/6259df03d2a149ceb4166612b336783d 2024-11-27T21:15:04,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/6259df03d2a149ceb4166612b336783d, entries=150, sequenceid=671, filesize=12.0 K 2024-11-27T21:15:04,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=137.04 KB/140334 for acbb7f305d2f548aa38b4230ffde5235 in 861ms, sequenceid=671, compaction requested=true 2024-11-27T21:15:04,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:04,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:15:04,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:04,574 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:04,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:15:04,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:04,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:04,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:04,574 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:04,577 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:04,577 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:04,577 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/B is initiating minor compaction (all files) 2024-11-27T21:15:04,577 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/A is initiating minor compaction (all files) 2024-11-27T21:15:04,577 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/B in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:04,577 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/A in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:04,577 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/34821fedb5bf43718dccda40a757da6e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/6ef55cda8c2e4ab3a4c672b11d32f053, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/532f713802074883b0d2adc1f54c4608] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=36.5 K 2024-11-27T21:15:04,577 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/e9a6047cdb0741d099392d371f88bafb, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8c3b621a8261421b8f58278b6f813e45, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/100585a8335c4e7f8015c337ca0f9d74] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=36.5 K 2024-11-27T21:15:04,577 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 34821fedb5bf43718dccda40a757da6e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=641, earliestPutTs=1732742101367 2024-11-27T21:15:04,577 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9a6047cdb0741d099392d371f88bafb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=641, earliestPutTs=1732742101367 2024-11-27T21:15:04,578 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ef55cda8c2e4ab3a4c672b11d32f053, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=659, earliestPutTs=1732742101539 2024-11-27T21:15:04,578 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c3b621a8261421b8f58278b6f813e45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=659, earliestPutTs=1732742101539 2024-11-27T21:15:04,578 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 532f713802074883b0d2adc1f54c4608, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1732742103705 2024-11-27T21:15:04,578 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 100585a8335c4e7f8015c337ca0f9d74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1732742103705 2024-11-27T21:15:04,586 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#A#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:04,586 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/d68acaf1cf874ea0bc2e880cae1d4f8c is 50, key is test_row_0/A:col10/1732742103711/Put/seqid=0 2024-11-27T21:15:04,587 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:04,587 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/6150742aecef400aacd73273d4435cee is 50, key is test_row_0/B:col10/1732742103711/Put/seqid=0 2024-11-27T21:15:04,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742134_1310 (size=12915) 2024-11-27T21:15:04,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742135_1311 (size=12915) 2024-11-27T21:15:04,616 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/6150742aecef400aacd73273d4435cee as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/6150742aecef400aacd73273d4435cee 2024-11-27T21:15:04,622 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/B of acbb7f305d2f548aa38b4230ffde5235 into 6150742aecef400aacd73273d4435cee(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:04,622 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:04,622 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/B, priority=13, startTime=1732742104574; duration=0sec 2024-11-27T21:15:04,622 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:04,622 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:B 2024-11-27T21:15:04,622 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:15:04,624 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:15:04,624 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/C is initiating minor compaction (all files) 2024-11-27T21:15:04,624 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/C in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:04,624 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c515bb5eac7847769a689a257b29755a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b5f872fbfa6d4b648cc3c23941028a16, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/3ca52e60831e4030adbd0ce07c5fcd69, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/6259df03d2a149ceb4166612b336783d] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=48.5 K 2024-11-27T21:15:04,625 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting c515bb5eac7847769a689a257b29755a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=623, earliestPutTs=1732742101271 2024-11-27T21:15:04,625 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting b5f872fbfa6d4b648cc3c23941028a16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=641, earliestPutTs=1732742101367 2024-11-27T21:15:04,626 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ca52e60831e4030adbd0ce07c5fcd69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=659, earliestPutTs=1732742101539 2024-11-27T21:15:04,626 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 6259df03d2a149ceb4166612b336783d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1732742103705 2024-11-27T21:15:04,637 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#C#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:04,637 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/3def572d21474bb2b77d0445bf9d0636 is 50, key is test_row_0/C:col10/1732742103711/Put/seqid=0 2024-11-27T21:15:04,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742136_1312 (size=12881) 2024-11-27T21:15:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:04,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=143.75 KB heapSize=405.88 KB 2024-11-27T21:15:04,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:04,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:04,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:04,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:04,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:04,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:04,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/4929423bb1944f66834bc6e0bab1b203 is 50, key is test_row_0/A:col10/1732742103866/Put/seqid=0 2024-11-27T21:15:04,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742137_1313 (size=14741) 2024-11-27T21:15:04,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 403 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742164989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:04,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742164990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:05,003 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/d68acaf1cf874ea0bc2e880cae1d4f8c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/d68acaf1cf874ea0bc2e880cae1d4f8c 2024-11-27T21:15:05,008 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/A of acbb7f305d2f548aa38b4230ffde5235 into d68acaf1cf874ea0bc2e880cae1d4f8c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:05,008 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:05,008 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/A, priority=13, startTime=1732742104573; duration=0sec 2024-11-27T21:15:05,008 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:05,008 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:A 2024-11-27T21:15:05,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742165009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:05,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:05,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742165009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:05,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742165009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:05,046 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/3def572d21474bb2b77d0445bf9d0636 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/3def572d21474bb2b77d0445bf9d0636 2024-11-27T21:15:05,050 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/C of acbb7f305d2f548aa38b4230ffde5235 into 3def572d21474bb2b77d0445bf9d0636(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:05,050 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:05,050 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/C, priority=12, startTime=1732742104574; duration=0sec 2024-11-27T21:15:05,050 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:05,050 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:C 2024-11-27T21:15:05,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 404 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742165099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:05,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742165099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:05,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 405 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742165308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:05,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742165309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:05,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=726 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/4929423bb1944f66834bc6e0bab1b203 2024-11-27T21:15:05,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/20ff4b28ffec4df1ac9e94bdbe67433e is 50, key is test_row_0/B:col10/1732742103866/Put/seqid=0 2024-11-27T21:15:05,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742138_1314 (size=12301) 2024-11-27T21:15:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T21:15:05,449 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:05,450 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:15:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-27T21:15:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T21:15:05,452 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:15:05,453 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:15:05,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:15:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T21:15:05,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T21:15:05,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:05,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:05,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:05,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:05,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:05,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 406 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742165618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742165618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:05,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T21:15:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:05,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T21:15:05,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=726 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/20ff4b28ffec4df1ac9e94bdbe67433e 2024-11-27T21:15:05,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/a942f96da6404fc1a888086490637069 is 50, key is test_row_0/C:col10/1732742103866/Put/seqid=0 2024-11-27T21:15:05,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742139_1315 (size=12301) 2024-11-27T21:15:05,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45.36 KB at sequenceid=726 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/a942f96da6404fc1a888086490637069 2024-11-27T21:15:05,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/4929423bb1944f66834bc6e0bab1b203 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/4929423bb1944f66834bc6e0bab1b203 2024-11-27T21:15:05,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/4929423bb1944f66834bc6e0bab1b203, entries=200, sequenceid=726, filesize=14.4 K 2024-11-27T21:15:05,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/20ff4b28ffec4df1ac9e94bdbe67433e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/20ff4b28ffec4df1ac9e94bdbe67433e 2024-11-27T21:15:05,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/20ff4b28ffec4df1ac9e94bdbe67433e, entries=150, sequenceid=726, filesize=12.0 K 2024-11-27T21:15:05,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/a942f96da6404fc1a888086490637069 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/a942f96da6404fc1a888086490637069 2024-11-27T21:15:05,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/a942f96da6404fc1a888086490637069, entries=150, sequenceid=726, filesize=12.0 K 2024-11-27T21:15:05,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.75 KB/147204, heapSize ~405.83 KB/415568, currentSize=40.25 KB/41220 for acbb7f305d2f548aa38b4230ffde5235 in 874ms, sequenceid=726, compaction requested=false 2024-11-27T21:15:05,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:05,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T21:15:05,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:05,913 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-27T21:15:05,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:05,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:05,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:05,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:05,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:05,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:05,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/018fe94e21454f1db6b9ebd1b2a1f1a7 is 50, key is test_row_0/A:col10/1732742104988/Put/seqid=0 2024-11-27T21:15:05,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742140_1316 (size=9857) 2024-11-27T21:15:05,923 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=737 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/018fe94e21454f1db6b9ebd1b2a1f1a7 2024-11-27T21:15:05,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/0ec5a08cb7094429bad393dfda63bea4 is 50, key is test_row_0/B:col10/1732742104988/Put/seqid=0 2024-11-27T21:15:05,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742141_1317 (size=9857) 2024-11-27T21:15:06,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:06,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742106056, max timestamp=1732742106073Num uniques 859; 2024-11-27T21:15:06,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742106056, max timestamp=1732742106073Num uniques 859; 2024-11-27T21:15:06,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:15:06,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:15:06,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742106056, max timestamp=1732742106073Num uniques 859; 2024-11-27T21:15:06,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:15:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T21:15:06,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742166105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732742106096, max timestamp=1732742106111Num uniques 1068; 2024-11-27T21:15:06,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-27T21:15:06,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732742106096, max timestamp=1732742106111Num uniques 1068; 2024-11-27T21:15:06,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-27T21:15:06,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732742106096, max timestamp=1732742106111Num uniques 1068; 2024-11-27T21:15:06,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-27T21:15:06,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742166149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 410 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742166151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742166150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742166153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742166219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742166258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 411 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742166269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742166269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742166269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,337 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=737 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/0ec5a08cb7094429bad393dfda63bea4 2024-11-27T21:15:06,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/f7b0d19133ff41b8b1bf7122520edfc0 is 50, key is test_row_0/C:col10/1732742104988/Put/seqid=0 2024-11-27T21:15:06,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742142_1318 (size=9857) 2024-11-27T21:15:06,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742166429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742166469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 412 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742166479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742166479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742166479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T21:15:06,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742166739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,749 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=737 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/f7b0d19133ff41b8b1bf7122520edfc0 2024-11-27T21:15:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/018fe94e21454f1db6b9ebd1b2a1f1a7 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/018fe94e21454f1db6b9ebd1b2a1f1a7 2024-11-27T21:15:06,757 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/018fe94e21454f1db6b9ebd1b2a1f1a7, entries=100, sequenceid=737, filesize=9.6 K 2024-11-27T21:15:06,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/0ec5a08cb7094429bad393dfda63bea4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/0ec5a08cb7094429bad393dfda63bea4 2024-11-27T21:15:06,762 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/0ec5a08cb7094429bad393dfda63bea4, entries=100, sequenceid=737, filesize=9.6 K 2024-11-27T21:15:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/f7b0d19133ff41b8b1bf7122520edfc0 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/f7b0d19133ff41b8b1bf7122520edfc0 2024-11-27T21:15:06,766 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/f7b0d19133ff41b8b1bf7122520edfc0, entries=100, sequenceid=737, filesize=9.6 K 2024-11-27T21:15:06,767 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for acbb7f305d2f548aa38b4230ffde5235 in 854ms, sequenceid=737, compaction requested=true 2024-11-27T21:15:06,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:06,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:06,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-27T21:15:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-27T21:15:06,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-27T21:15:06,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3150 sec 2024-11-27T21:15:06,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.3210 sec 2024-11-27T21:15:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:06,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-27T21:15:06,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:06,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:15:06,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:06,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:15:06,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:06,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:15:06,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/ddd69e795e564abdafd115504b6d3006 is 50, key is test_row_0/A:col10/1732742106139/Put/seqid=0 2024-11-27T21:15:06,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742143_1319 (size=14741) 2024-11-27T21:15:06,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=792 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/ddd69e795e564abdafd115504b6d3006 2024-11-27T21:15:06,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 414 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742166795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/9cf0a4b48abd4c2f9b82582cef95f562 is 50, key is test_row_0/B:col10/1732742106139/Put/seqid=0 2024-11-27T21:15:06,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742166797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742166797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742166798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742144_1320 (size=12301) 2024-11-27T21:15:06,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=792 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/9cf0a4b48abd4c2f9b82582cef95f562 2024-11-27T21:15:06,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/6336a7bfa23e4f47bf7317b431ddb77f is 50, key is test_row_0/C:col10/1732742106139/Put/seqid=0 2024-11-27T21:15:06,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742145_1321 (size=12301) 2024-11-27T21:15:06,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 415 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742166909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742166909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742166909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:06,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742166910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:06,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:06,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:06,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:07,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742167120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:07,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 416 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742167120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742167120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:07,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742167120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:07,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=792 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/6336a7bfa23e4f47bf7317b431ddb77f 2024-11-27T21:15:07,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/ddd69e795e564abdafd115504b6d3006 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/ddd69e795e564abdafd115504b6d3006 2024-11-27T21:15:07,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/ddd69e795e564abdafd115504b6d3006, entries=200, sequenceid=792, filesize=14.4 K 2024-11-27T21:15:07,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/9cf0a4b48abd4c2f9b82582cef95f562 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/9cf0a4b48abd4c2f9b82582cef95f562 2024-11-27T21:15:07,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/9cf0a4b48abd4c2f9b82582cef95f562, entries=150, sequenceid=792, filesize=12.0 K 2024-11-27T21:15:07,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/6336a7bfa23e4f47bf7317b431ddb77f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/6336a7bfa23e4f47bf7317b431ddb77f 2024-11-27T21:15:07,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/6336a7bfa23e4f47bf7317b431ddb77f, entries=150, sequenceid=792, filesize=12.0 K 2024-11-27T21:15:07,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for acbb7f305d2f548aa38b4230ffde5235 in 465ms, sequenceid=792, compaction requested=true 2024-11-27T21:15:07,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:07,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:15:07,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:07,246 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:15:07,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:15:07,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:07,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:07,246 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:15:07,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:07,247 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52254 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:15:07,247 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47374 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:15:07,247 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/A is initiating minor compaction (all files) 2024-11-27T21:15:07,247 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/B is initiating minor compaction (all files) 2024-11-27T21:15:07,247 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/A in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:07,247 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/B in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:07,247 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/d68acaf1cf874ea0bc2e880cae1d4f8c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/4929423bb1944f66834bc6e0bab1b203, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/018fe94e21454f1db6b9ebd1b2a1f1a7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/ddd69e795e564abdafd115504b6d3006] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=51.0 K 2024-11-27T21:15:07,247 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/6150742aecef400aacd73273d4435cee, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/20ff4b28ffec4df1ac9e94bdbe67433e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/0ec5a08cb7094429bad393dfda63bea4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/9cf0a4b48abd4c2f9b82582cef95f562] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=46.3 K 2024-11-27T21:15:07,248 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting d68acaf1cf874ea0bc2e880cae1d4f8c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1732742103705 2024-11-27T21:15:07,248 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 6150742aecef400aacd73273d4435cee, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1732742103705 2024-11-27T21:15:07,248 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4929423bb1944f66834bc6e0bab1b203, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=726, earliestPutTs=1732742103857 2024-11-27T21:15:07,248 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 20ff4b28ffec4df1ac9e94bdbe67433e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=726, earliestPutTs=1732742103864 2024-11-27T21:15:07,248 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 018fe94e21454f1db6b9ebd1b2a1f1a7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=737, earliestPutTs=1732742104988 2024-11-27T21:15:07,249 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ec5a08cb7094429bad393dfda63bea4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=737, earliestPutTs=1732742104988 2024-11-27T21:15:07,249 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddd69e795e564abdafd115504b6d3006, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=792, earliestPutTs=1732742106139 2024-11-27T21:15:07,249 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cf0a4b48abd4c2f9b82582cef95f562, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=792, earliestPutTs=1732742106139 2024-11-27T21:15:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:07,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:15:07,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:07,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:07,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:07,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:07,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:07,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:07,259 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#A#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:07,260 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/8903513aa1ce411e8c68fb557e3ca43d is 50, key is test_row_0/A:col10/1732742106139/Put/seqid=0 2024-11-27T21:15:07,262 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#B#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:07,263 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/5e60a7ee8b5149d198b400c5da25c9ef is 50, key is test_row_0/B:col10/1732742106139/Put/seqid=0 2024-11-27T21:15:07,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/53fb8b16ece14da681cc45a5985a3481 is 50, key is test_row_0/A:col10/1732742107251/Put/seqid=0 2024-11-27T21:15:07,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742146_1322 (size=13051) 2024-11-27T21:15:07,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742147_1323 (size=13051) 2024-11-27T21:15:07,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742148_1324 (size=12301) 2024-11-27T21:15:07,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=802 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/53fb8b16ece14da681cc45a5985a3481 2024-11-27T21:15:07,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/a991de8dbb1e40fe8f9bb57e552ed0c6 is 50, key is test_row_0/B:col10/1732742107251/Put/seqid=0 2024-11-27T21:15:07,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742149_1325 (size=12301) 2024-11-27T21:15:07,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=802 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/a991de8dbb1e40fe8f9bb57e552ed0c6 2024-11-27T21:15:07,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/9666dfec0a064229ab8aa99835fcec98 is 50, key is test_row_0/C:col10/1732742107251/Put/seqid=0 2024-11-27T21:15:07,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742150_1326 (size=12301) 2024-11-27T21:15:07,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=802 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/9666dfec0a064229ab8aa99835fcec98 2024-11-27T21:15:07,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/53fb8b16ece14da681cc45a5985a3481 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/53fb8b16ece14da681cc45a5985a3481 2024-11-27T21:15:07,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/53fb8b16ece14da681cc45a5985a3481, entries=150, sequenceid=802, filesize=12.0 K 2024-11-27T21:15:07,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/a991de8dbb1e40fe8f9bb57e552ed0c6 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/a991de8dbb1e40fe8f9bb57e552ed0c6 2024-11-27T21:15:07,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/a991de8dbb1e40fe8f9bb57e552ed0c6, entries=150, sequenceid=802, filesize=12.0 K 2024-11-27T21:15:07,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/9666dfec0a064229ab8aa99835fcec98 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/9666dfec0a064229ab8aa99835fcec98 2024-11-27T21:15:07,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/9666dfec0a064229ab8aa99835fcec98, entries=150, sequenceid=802, filesize=12.0 K 2024-11-27T21:15:07,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=60.38 KB/61830 for acbb7f305d2f548aa38b4230ffde5235 in 56ms, sequenceid=802, compaction requested=true 2024-11-27T21:15:07,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:07,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:A, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:07,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T21:15:07,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:B, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:07,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-27T21:15:07,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:07,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-27T21:15:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:07,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-27T21:15:07,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:07,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:07,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:07,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:07,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:07,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:07,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/df7549a162144001babd4a5abf14889d is 50, key is test_row_0/A:col10/1732742107299/Put/seqid=0 2024-11-27T21:15:07,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742151_1327 (size=14741) 2024-11-27T21:15:07,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742167415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:07,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742167429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:07,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:07,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 417 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742167429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:07,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:07,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742167429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:07,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742167439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:07,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:07,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742167529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:07,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T21:15:07,610 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:07,610 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:15:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-27T21:15:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T21:15:07,612 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:15:07,612 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:15:07,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:15:07,674 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/8903513aa1ce411e8c68fb557e3ca43d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8903513aa1ce411e8c68fb557e3ca43d 2024-11-27T21:15:07,674 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/5e60a7ee8b5149d198b400c5da25c9ef as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/5e60a7ee8b5149d198b400c5da25c9ef 2024-11-27T21:15:07,679 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/A of acbb7f305d2f548aa38b4230ffde5235 into 8903513aa1ce411e8c68fb557e3ca43d(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:07,679 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:07,679 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/A, priority=12, startTime=1732742107246; duration=0sec 2024-11-27T21:15:07,679 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-27T21:15:07,679 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:A 2024-11-27T21:15:07,679 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:A 2024-11-27T21:15:07,679 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T21:15:07,679 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/B of acbb7f305d2f548aa38b4230ffde5235 into 5e60a7ee8b5149d198b400c5da25c9ef(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:07,679 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:07,679 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/B, priority=12, startTime=1732742107246; duration=0sec 2024-11-27T21:15:07,679 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-27T21:15:07,679 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:B 2024-11-27T21:15:07,679 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:B 2024-11-27T21:15:07,681 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59641 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T21:15:07,681 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/C is initiating minor compaction (all files) 2024-11-27T21:15:07,681 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/C in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:07,681 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/3def572d21474bb2b77d0445bf9d0636, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/a942f96da6404fc1a888086490637069, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/f7b0d19133ff41b8b1bf7122520edfc0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/6336a7bfa23e4f47bf7317b431ddb77f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/9666dfec0a064229ab8aa99835fcec98] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=58.2 K 2024-11-27T21:15:07,681 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-27T21:15:07,681 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T21:15:07,681 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T21:15:07,681 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. because compaction request was cancelled 2024-11-27T21:15:07,681 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:C 2024-11-27T21:15:07,681 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T21:15:07,682 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3def572d21474bb2b77d0445bf9d0636, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1732742103705 2024-11-27T21:15:07,682 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting a942f96da6404fc1a888086490637069, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=726, earliestPutTs=1732742103864 2024-11-27T21:15:07,683 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7b0d19133ff41b8b1bf7122520edfc0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=737, earliestPutTs=1732742104988 2024-11-27T21:15:07,683 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T21:15:07,683 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T21:15:07,683 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. because compaction request was cancelled 2024-11-27T21:15:07,683 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:B 2024-11-27T21:15:07,683 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T21:15:07,683 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T21:15:07,683 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T21:15:07,684 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6336a7bfa23e4f47bf7317b431ddb77f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=792, earliestPutTs=1732742106139 2024-11-27T21:15:07,684 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. because compaction request was cancelled 2024-11-27T21:15:07,684 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:A 2024-11-27T21:15:07,684 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9666dfec0a064229ab8aa99835fcec98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1732742106796 2024-11-27T21:15:07,699 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#C#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:07,700 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/e02c8cf2f03a4144a3da464d0633ab14 is 50, key is test_row_0/C:col10/1732742107251/Put/seqid=0 2024-11-27T21:15:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T21:15:07,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=815 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/df7549a162144001babd4a5abf14889d 2024-11-27T21:15:07,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/b3ca71a87e7741e7b8ccdcc77086dab8 is 50, key is test_row_0/B:col10/1732742107299/Put/seqid=0 2024-11-27T21:15:07,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742167739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:07,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742153_1329 (size=12301) 2024-11-27T21:15:07,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742152_1328 (size=13051) 2024-11-27T21:15:07,761 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/e02c8cf2f03a4144a3da464d0633ab14 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/e02c8cf2f03a4144a3da464d0633ab14 2024-11-27T21:15:07,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T21:15:07,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:07,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:07,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:07,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:07,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:07,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:07,769 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/C of acbb7f305d2f548aa38b4230ffde5235 into e02c8cf2f03a4144a3da464d0633ab14(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:07,769 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:07,769 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/C, priority=11, startTime=1732742107313; duration=0sec 2024-11-27T21:15:07,770 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:07,770 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:C 2024-11-27T21:15:07,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T21:15:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:07,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:07,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:07,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T21:15:07,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:07,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742167939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:07,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 418 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742167940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:07,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742167940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:07,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:07,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742167949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:08,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:08,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742168048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:08,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T21:15:08,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:08,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:08,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:08,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:08,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:08,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=815 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/b3ca71a87e7741e7b8ccdcc77086dab8 2024-11-27T21:15:08,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/c30fad3e4cf74c589290bd7242b31963 is 50, key is test_row_0/C:col10/1732742107299/Put/seqid=0 2024-11-27T21:15:08,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742154_1330 (size=12301) 2024-11-27T21:15:08,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T21:15:08,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:08,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:08,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:08,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:08,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T21:15:08,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T21:15:08,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:08,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:08,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:08,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:08,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:08,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T21:15:08,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:08,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:08,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:08,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:08,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:08,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742168559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:08,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=815 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/c30fad3e4cf74c589290bd7242b31963 2024-11-27T21:15:08,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/df7549a162144001babd4a5abf14889d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/df7549a162144001babd4a5abf14889d 2024-11-27T21:15:08,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/df7549a162144001babd4a5abf14889d, entries=200, sequenceid=815, filesize=14.4 K 2024-11-27T21:15:08,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/b3ca71a87e7741e7b8ccdcc77086dab8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/b3ca71a87e7741e7b8ccdcc77086dab8 2024-11-27T21:15:08,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/b3ca71a87e7741e7b8ccdcc77086dab8, entries=150, sequenceid=815, filesize=12.0 K 2024-11-27T21:15:08,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/c30fad3e4cf74c589290bd7242b31963 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c30fad3e4cf74c589290bd7242b31963 2024-11-27T21:15:08,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c30fad3e4cf74c589290bd7242b31963, entries=150, sequenceid=815, filesize=12.0 K 2024-11-27T21:15:08,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for acbb7f305d2f548aa38b4230ffde5235 in 1271ms, sequenceid=815, compaction requested=true 2024-11-27T21:15:08,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:08,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:15:08,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:08,585 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:08,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:15:08,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:08,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:08,585 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:08,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:08,586 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:08,586 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:08,586 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/A is initiating minor compaction (all files) 2024-11-27T21:15:08,586 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/B is initiating minor compaction (all files) 2024-11-27T21:15:08,586 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/A in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:08,586 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/B in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:08,586 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8903513aa1ce411e8c68fb557e3ca43d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/53fb8b16ece14da681cc45a5985a3481, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/df7549a162144001babd4a5abf14889d] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=39.2 K 2024-11-27T21:15:08,586 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/5e60a7ee8b5149d198b400c5da25c9ef, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/a991de8dbb1e40fe8f9bb57e552ed0c6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/b3ca71a87e7741e7b8ccdcc77086dab8] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=36.8 K 2024-11-27T21:15:08,586 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8903513aa1ce411e8c68fb557e3ca43d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=792, earliestPutTs=1732742106139 2024-11-27T21:15:08,586 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e60a7ee8b5149d198b400c5da25c9ef, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=792, earliestPutTs=1732742106139 2024-11-27T21:15:08,586 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53fb8b16ece14da681cc45a5985a3481, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1732742106796 2024-11-27T21:15:08,586 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting a991de8dbb1e40fe8f9bb57e552ed0c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1732742106796 2024-11-27T21:15:08,587 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting df7549a162144001babd4a5abf14889d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=815, earliestPutTs=1732742107281 2024-11-27T21:15:08,587 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting b3ca71a87e7741e7b8ccdcc77086dab8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=815, earliestPutTs=1732742107299 2024-11-27T21:15:08,593 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#B#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:08,594 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/f44394a4b25b434ea74f3de91cc8e6d6 is 50, key is test_row_0/B:col10/1732742107299/Put/seqid=0 2024-11-27T21:15:08,596 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#A#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:08,596 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/622fecec151147b9831499b3c8e320cb is 50, key is test_row_0/A:col10/1732742107299/Put/seqid=0 2024-11-27T21:15:08,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742155_1331 (size=13153) 2024-11-27T21:15:08,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742156_1332 (size=13153) 2024-11-27T21:15:08,605 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/f44394a4b25b434ea74f3de91cc8e6d6 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/f44394a4b25b434ea74f3de91cc8e6d6 2024-11-27T21:15:08,610 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/B of acbb7f305d2f548aa38b4230ffde5235 into f44394a4b25b434ea74f3de91cc8e6d6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:08,610 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:08,610 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/B, priority=13, startTime=1732742108585; duration=0sec 2024-11-27T21:15:08,610 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:08,610 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:B 2024-11-27T21:15:08,610 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T21:15:08,611 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T21:15:08,611 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T21:15:08,611 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. because compaction request was cancelled 2024-11-27T21:15:08,611 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:C 2024-11-27T21:15:08,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T21:15:08,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:08,686 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T21:15:08,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:08,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:08,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:08,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:08,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:08,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:08,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/c549e4bb36d24842bf3da228d05705f8 is 50, key is test_row_0/A:col10/1732742107390/Put/seqid=0 2024-11-27T21:15:08,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742157_1333 (size=12301) 2024-11-27T21:15:08,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T21:15:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:08,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:08,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742168969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:08,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:08,973 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_0' in acbb7f305d2f548aa38b4230ffde5235 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:15:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 421 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742168970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:08,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742168971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:08,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742168972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:09,006 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/622fecec151147b9831499b3c8e320cb as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/622fecec151147b9831499b3c8e320cb 2024-11-27T21:15:09,010 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/A of acbb7f305d2f548aa38b4230ffde5235 into 622fecec151147b9831499b3c8e320cb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:09,010 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:09,010 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/A, priority=13, startTime=1732742108585; duration=0sec 2024-11-27T21:15:09,010 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:09,010 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:A 2024-11-27T21:15:09,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 422 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742169079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:09,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742169079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:09,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742169079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:09,095 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=838 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/c549e4bb36d24842bf3da228d05705f8 2024-11-27T21:15:09,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/00ec20d8af4d4f91b09051ab4412a0f0 is 50, key is test_row_0/B:col10/1732742107390/Put/seqid=0 2024-11-27T21:15:09,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742158_1334 (size=12301) 2024-11-27T21:15:09,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742169289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:09,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 423 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742169289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742169289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:09,508 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=838 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/00ec20d8af4d4f91b09051ab4412a0f0 2024-11-27T21:15:09,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/b33bc6d65cac49c1a3c3e53466fe981d is 50, key is test_row_0/C:col10/1732742107390/Put/seqid=0 2024-11-27T21:15:09,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742159_1335 (size=12301) 2024-11-27T21:15:09,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742169579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:09,583 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_0' in acbb7f305d2f548aa38b4230ffde5235 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:15:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:09,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742169599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:09,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 424 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742169599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742169599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T21:15:09,921 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=838 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/b33bc6d65cac49c1a3c3e53466fe981d 2024-11-27T21:15:09,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/c549e4bb36d24842bf3da228d05705f8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/c549e4bb36d24842bf3da228d05705f8 2024-11-27T21:15:09,929 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/c549e4bb36d24842bf3da228d05705f8, entries=150, sequenceid=838, filesize=12.0 K 2024-11-27T21:15:09,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/00ec20d8af4d4f91b09051ab4412a0f0 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/00ec20d8af4d4f91b09051ab4412a0f0 2024-11-27T21:15:09,934 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/00ec20d8af4d4f91b09051ab4412a0f0, entries=150, sequenceid=838, filesize=12.0 K 2024-11-27T21:15:09,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/b33bc6d65cac49c1a3c3e53466fe981d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b33bc6d65cac49c1a3c3e53466fe981d 2024-11-27T21:15:09,938 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b33bc6d65cac49c1a3c3e53466fe981d, entries=150, sequenceid=838, filesize=12.0 K 2024-11-27T21:15:09,939 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for acbb7f305d2f548aa38b4230ffde5235 in 1254ms, sequenceid=838, compaction requested=true 2024-11-27T21:15:09,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:09,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:09,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-27T21:15:09,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-27T21:15:09,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-27T21:15:09,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3270 sec 2024-11-27T21:15:09,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.3330 sec 2024-11-27T21:15:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:10,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T21:15:10,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:10,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:10,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:10,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:10,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:10,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:10,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/3d2e84f8ce96413ea0aeb2259dbc1041 is 50, key is test_row_0/A:col10/1732742110109/Put/seqid=0 2024-11-27T21:15:10,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742160_1336 (size=12301) 2024-11-27T21:15:10,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=853 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/3d2e84f8ce96413ea0aeb2259dbc1041 2024-11-27T21:15:10,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/80d5d2504d9a44ebb23d81df5d79ff57 is 50, key is test_row_0/B:col10/1732742110109/Put/seqid=0 2024-11-27T21:15:10,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:10,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742170137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:10,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742161_1337 (size=12301) 2024-11-27T21:15:10,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742170143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:10,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 431 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742170143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:10,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:10,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742170249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:10,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742170259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:10,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 432 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742170259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:10,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742170459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:10,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742170469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:10,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 433 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742170469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:10,503 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T21:15:10,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=853 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/80d5d2504d9a44ebb23d81df5d79ff57 2024-11-27T21:15:10,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/44ca63e6c25142a7a539ffab520db0de is 50, key is test_row_0/C:col10/1732742110109/Put/seqid=0 2024-11-27T21:15:10,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742162_1338 (size=12301) 2024-11-27T21:15:10,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742170769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:10,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742170778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:10,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 434 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742170779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:10,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=853 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/44ca63e6c25142a7a539ffab520db0de 2024-11-27T21:15:10,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/3d2e84f8ce96413ea0aeb2259dbc1041 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/3d2e84f8ce96413ea0aeb2259dbc1041 2024-11-27T21:15:10,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/3d2e84f8ce96413ea0aeb2259dbc1041, entries=150, sequenceid=853, filesize=12.0 K 2024-11-27T21:15:10,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/80d5d2504d9a44ebb23d81df5d79ff57 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/80d5d2504d9a44ebb23d81df5d79ff57 2024-11-27T21:15:10,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/80d5d2504d9a44ebb23d81df5d79ff57, entries=150, sequenceid=853, filesize=12.0 K 2024-11-27T21:15:10,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/44ca63e6c25142a7a539ffab520db0de as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/44ca63e6c25142a7a539ffab520db0de 2024-11-27T21:15:10,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/44ca63e6c25142a7a539ffab520db0de, entries=150, sequenceid=853, filesize=12.0 K 2024-11-27T21:15:10,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for acbb7f305d2f548aa38b4230ffde5235 in 867ms, sequenceid=853, compaction requested=true 2024-11-27T21:15:10,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:10,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:15:10,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:10,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:15:10,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:10,978 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:10,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:10,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:10,978 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:10,979 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:10,979 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:10,979 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/B is initiating minor compaction (all files) 2024-11-27T21:15:10,979 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/A is initiating minor compaction (all files) 2024-11-27T21:15:10,979 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/B in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:10,979 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/A in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:10,979 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/f44394a4b25b434ea74f3de91cc8e6d6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/00ec20d8af4d4f91b09051ab4412a0f0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/80d5d2504d9a44ebb23d81df5d79ff57] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=36.9 K 2024-11-27T21:15:10,979 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/622fecec151147b9831499b3c8e320cb, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/c549e4bb36d24842bf3da228d05705f8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/3d2e84f8ce96413ea0aeb2259dbc1041] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=36.9 K 2024-11-27T21:15:10,980 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting f44394a4b25b434ea74f3de91cc8e6d6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=815, earliestPutTs=1732742107299 2024-11-27T21:15:10,980 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 622fecec151147b9831499b3c8e320cb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=815, earliestPutTs=1732742107299 2024-11-27T21:15:10,980 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 00ec20d8af4d4f91b09051ab4412a0f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=838, earliestPutTs=1732742107390 2024-11-27T21:15:10,981 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting c549e4bb36d24842bf3da228d05705f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=838, earliestPutTs=1732742107390 2024-11-27T21:15:10,981 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 80d5d2504d9a44ebb23d81df5d79ff57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=853, earliestPutTs=1732742108967 2024-11-27T21:15:10,981 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d2e84f8ce96413ea0aeb2259dbc1041, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=853, earliestPutTs=1732742108967 2024-11-27T21:15:10,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:10,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-27T21:15:10,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:10,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:10,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:10,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:10,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:10,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:10,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/0aeea06bbe584522a3c784b5afea022e is 50, key is test_row_0/A:col10/1732742110980/Put/seqid=0 2024-11-27T21:15:10,990 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#A#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:10,990 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#B#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:10,990 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/37e687cb5b5d408fa7d3ddd2279bf3e1 is 50, key is test_row_0/A:col10/1732742110109/Put/seqid=0 2024-11-27T21:15:10,991 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/5aacd33604b142b69e7679633e6b184a is 50, key is test_row_0/B:col10/1732742110109/Put/seqid=0 2024-11-27T21:15:11,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742164_1340 (size=13255) 2024-11-27T21:15:11,012 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/37e687cb5b5d408fa7d3ddd2279bf3e1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/37e687cb5b5d408fa7d3ddd2279bf3e1 2024-11-27T21:15:11,017 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/A of acbb7f305d2f548aa38b4230ffde5235 into 37e687cb5b5d408fa7d3ddd2279bf3e1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:11,017 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:11,017 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/A, priority=13, startTime=1732742110978; duration=0sec 2024-11-27T21:15:11,018 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:11,018 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:A 2024-11-27T21:15:11,018 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:15:11,019 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:15:11,019 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/C is initiating minor compaction (all files) 2024-11-27T21:15:11,019 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/C in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:11,019 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/e02c8cf2f03a4144a3da464d0633ab14, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c30fad3e4cf74c589290bd7242b31963, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b33bc6d65cac49c1a3c3e53466fe981d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/44ca63e6c25142a7a539ffab520db0de] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=48.8 K 2024-11-27T21:15:11,019 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting e02c8cf2f03a4144a3da464d0633ab14, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1732742106796 2024-11-27T21:15:11,020 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting c30fad3e4cf74c589290bd7242b31963, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=815, earliestPutTs=1732742107299 2024-11-27T21:15:11,020 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting b33bc6d65cac49c1a3c3e53466fe981d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=838, earliestPutTs=1732742107390 2024-11-27T21:15:11,021 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44ca63e6c25142a7a539ffab520db0de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=853, earliestPutTs=1732742108967 2024-11-27T21:15:11,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742165_1341 (size=13255) 2024-11-27T21:15:11,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742163_1339 (size=14741) 2024-11-27T21:15:11,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=873 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/0aeea06bbe584522a3c784b5afea022e 2024-11-27T21:15:11,035 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#C#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:11,036 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/91878d14210e48999a11f16f2c1e28f2 is 50, key is test_row_0/C:col10/1732742110109/Put/seqid=0 2024-11-27T21:15:11,039 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/5aacd33604b142b69e7679633e6b184a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/5aacd33604b142b69e7679633e6b184a 2024-11-27T21:15:11,045 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/B of acbb7f305d2f548aa38b4230ffde5235 into 5aacd33604b142b69e7679633e6b184a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:11,045 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:11,045 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/B, priority=13, startTime=1732742110978; duration=0sec 2024-11-27T21:15:11,045 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:11,045 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:B 2024-11-27T21:15:11,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/88126383a4bd4af69baa1d796daa6fea is 50, key is test_row_0/B:col10/1732742110980/Put/seqid=0 2024-11-27T21:15:11,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742171045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:11,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742166_1342 (size=13187) 2024-11-27T21:15:11,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742167_1343 (size=12301) 2024-11-27T21:15:11,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742171159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:11,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:11,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:11,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:11,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742171278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:11,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742171289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:11,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 435 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742171289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:11,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:11,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:11,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:11,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:11,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:11,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:11,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742171368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:11,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:11,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:11,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:11,463 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/91878d14210e48999a11f16f2c1e28f2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/91878d14210e48999a11f16f2c1e28f2 2024-11-27T21:15:11,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=873 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/88126383a4bd4af69baa1d796daa6fea 2024-11-27T21:15:11,468 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/C of acbb7f305d2f548aa38b4230ffde5235 into 91878d14210e48999a11f16f2c1e28f2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:11,468 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:11,468 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/C, priority=12, startTime=1732742110978; duration=0sec 2024-11-27T21:15:11,468 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:11,468 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:C 2024-11-27T21:15:11,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/434247733a284708aec9b2db9a23c458 is 50, key is test_row_0/C:col10/1732742110980/Put/seqid=0 2024-11-27T21:15:11,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742168_1344 (size=12301) 2024-11-27T21:15:11,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53968 deadline: 1732742171599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:11,603 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_0' in acbb7f305d2f548aa38b4230ffde5235 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:15:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:11,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742171680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T21:15:11,769 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:11,770 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:15:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-27T21:15:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T21:15:11,775 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:15:11,776 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:15:11,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:15:11,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=873 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/434247733a284708aec9b2db9a23c458 2024-11-27T21:15:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T21:15:11,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/0aeea06bbe584522a3c784b5afea022e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/0aeea06bbe584522a3c784b5afea022e 2024-11-27T21:15:11,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/0aeea06bbe584522a3c784b5afea022e, entries=200, sequenceid=873, filesize=14.4 K 2024-11-27T21:15:11,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/88126383a4bd4af69baa1d796daa6fea as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/88126383a4bd4af69baa1d796daa6fea 2024-11-27T21:15:11,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/88126383a4bd4af69baa1d796daa6fea, entries=150, sequenceid=873, filesize=12.0 K 2024-11-27T21:15:11,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/434247733a284708aec9b2db9a23c458 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/434247733a284708aec9b2db9a23c458 2024-11-27T21:15:11,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/434247733a284708aec9b2db9a23c458, entries=150, sequenceid=873, filesize=12.0 K 2024-11-27T21:15:11,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for acbb7f305d2f548aa38b4230ffde5235 in 934ms, sequenceid=873, compaction requested=false 2024-11-27T21:15:11,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:11,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-27T21:15:11,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:11,931 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-27T21:15:11,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:11,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:11,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:11,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:11,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:11,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:11,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/aae1424948af4c698ddb8e83ed9231dd is 50, key is test_row_0/A:col10/1732742111043/Put/seqid=0 2024-11-27T21:15:11,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742169_1345 (size=12301) 2024-11-27T21:15:11,959 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=888 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/aae1424948af4c698ddb8e83ed9231dd 2024-11-27T21:15:11,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/1b7bf6b09fc64684a5ee2e65ef9ad560 is 50, key is test_row_0/B:col10/1732742111043/Put/seqid=0 2024-11-27T21:15:11,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742170_1346 (size=12301) 2024-11-27T21:15:11,979 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=888 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/1b7bf6b09fc64684a5ee2e65ef9ad560 2024-11-27T21:15:12,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/a3208746fbb241c3b57e997832353482 is 50, key is test_row_0/C:col10/1732742111043/Put/seqid=0 2024-11-27T21:15:12,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742171_1347 (size=12301) 2024-11-27T21:15:12,043 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=888 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/a3208746fbb241c3b57e997832353482 2024-11-27T21:15:12,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/aae1424948af4c698ddb8e83ed9231dd as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/aae1424948af4c698ddb8e83ed9231dd 2024-11-27T21:15:12,054 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/aae1424948af4c698ddb8e83ed9231dd, entries=150, sequenceid=888, filesize=12.0 K 2024-11-27T21:15:12,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/1b7bf6b09fc64684a5ee2e65ef9ad560 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/1b7bf6b09fc64684a5ee2e65ef9ad560 2024-11-27T21:15:12,059 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/1b7bf6b09fc64684a5ee2e65ef9ad560, entries=150, sequenceid=888, filesize=12.0 K 2024-11-27T21:15:12,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/a3208746fbb241c3b57e997832353482 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/a3208746fbb241c3b57e997832353482 2024-11-27T21:15:12,066 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/a3208746fbb241c3b57e997832353482, entries=150, sequenceid=888, filesize=12.0 K 2024-11-27T21:15:12,067 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for acbb7f305d2f548aa38b4230ffde5235 in 136ms, sequenceid=888, compaction requested=true 2024-11-27T21:15:12,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:12,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:12,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-27T21:15:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-27T21:15:12,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-27T21:15:12,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 292 msec 2024-11-27T21:15:12,072 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 301 msec 2024-11-27T21:15:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T21:15:12,090 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:12,090 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:15:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-27T21:15:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T21:15:12,092 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:15:12,093 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:15:12,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:15:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T21:15:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:12,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:15:12,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:12,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:12,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:12,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:12,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:12,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:12,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/e86065a5b7ab4a55b522b781f9308aa4 is 50, key is test_row_0/A:col10/1732742112201/Put/seqid=0 2024-11-27T21:15:12,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742172_1348 (size=14741) 2024-11-27T21:15:12,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=898 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/e86065a5b7ab4a55b522b781f9308aa4 2024-11-27T21:15:12,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/ecda8cff631949638b7997897d1f2575 is 50, key is test_row_0/B:col10/1732742112201/Put/seqid=0 2024-11-27T21:15:12,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742173_1349 (size=12301) 2024-11-27T21:15:12,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=898 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/ecda8cff631949638b7997897d1f2575 2024-11-27T21:15:12,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/13296fff43e04895a894b41929e162b2 is 50, key is test_row_0/C:col10/1732742112201/Put/seqid=0 2024-11-27T21:15:12,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742174_1350 (size=12301) 2024-11-27T21:15:12,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:15:12,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:12,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:12,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:12,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:12,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:12,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=898 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/13296fff43e04895a894b41929e162b2 2024-11-27T21:15:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:12,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/e86065a5b7ab4a55b522b781f9308aa4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/e86065a5b7ab4a55b522b781f9308aa4 2024-11-27T21:15:12,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/e86065a5b7ab4a55b522b781f9308aa4, entries=200, sequenceid=898, filesize=14.4 K 2024-11-27T21:15:12,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/ecda8cff631949638b7997897d1f2575 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/ecda8cff631949638b7997897d1f2575 2024-11-27T21:15:12,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/ecda8cff631949638b7997897d1f2575, entries=150, sequenceid=898, filesize=12.0 K 2024-11-27T21:15:12,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/13296fff43e04895a894b41929e162b2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/13296fff43e04895a894b41929e162b2 2024-11-27T21:15:12,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/13296fff43e04895a894b41929e162b2, entries=150, sequenceid=898, filesize=12.0 K 2024-11-27T21:15:12,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=67.09 KB/68700 for acbb7f305d2f548aa38b4230ffde5235 in 59ms, sequenceid=898, compaction requested=true 2024-11-27T21:15:12,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:12,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:15:12,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:12,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:15:12,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:12,267 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:15:12,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acbb7f305d2f548aa38b4230ffde5235:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:12,268 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:15:12,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:12,269 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55038 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:15:12,269 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:15:12,269 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/A is initiating minor compaction (all files) 2024-11-27T21:15:12,269 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/B is initiating minor compaction (all files) 2024-11-27T21:15:12,269 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/A in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:12,269 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/B in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:12,269 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/37e687cb5b5d408fa7d3ddd2279bf3e1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/0aeea06bbe584522a3c784b5afea022e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/aae1424948af4c698ddb8e83ed9231dd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/e86065a5b7ab4a55b522b781f9308aa4] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=53.7 K 2024-11-27T21:15:12,269 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/5aacd33604b142b69e7679633e6b184a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/88126383a4bd4af69baa1d796daa6fea, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/1b7bf6b09fc64684a5ee2e65ef9ad560, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/ecda8cff631949638b7997897d1f2575] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=49.0 K 2024-11-27T21:15:12,269 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37e687cb5b5d408fa7d3ddd2279bf3e1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=853, earliestPutTs=1732742108967 2024-11-27T21:15:12,270 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aeea06bbe584522a3c784b5afea022e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=873, earliestPutTs=1732742110117 2024-11-27T21:15:12,270 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aacd33604b142b69e7679633e6b184a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=853, earliestPutTs=1732742108967 2024-11-27T21:15:12,270 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting aae1424948af4c698ddb8e83ed9231dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=888, earliestPutTs=1732742110996 2024-11-27T21:15:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:12,270 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting e86065a5b7ab4a55b522b781f9308aa4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=898, earliestPutTs=1732742112195 2024-11-27T21:15:12,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T21:15:12,270 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 88126383a4bd4af69baa1d796daa6fea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=873, earliestPutTs=1732742110117 2024-11-27T21:15:12,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:12,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:12,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:12,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:12,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:12,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:12,271 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b7bf6b09fc64684a5ee2e65ef9ad560, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=888, earliestPutTs=1732742110996 2024-11-27T21:15:12,272 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting ecda8cff631949638b7997897d1f2575, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=898, earliestPutTs=1732742112195 2024-11-27T21:15:12,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/02853a04ddb64b6baaaa75448501ce5c is 50, key is test_row_0/A:col10/1732742112234/Put/seqid=0 2024-11-27T21:15:12,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742175_1351 (size=14741) 2024-11-27T21:15:12,282 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#A#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:12,283 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/15334cb17f5c403a9baa1fe5884d1e85 is 50, key is test_row_0/A:col10/1732742112201/Put/seqid=0 2024-11-27T21:15:12,287 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#B#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:12,287 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/320e5e6b7e094d76b079e0658e8bd0c1 is 50, key is test_row_0/B:col10/1732742112201/Put/seqid=0 2024-11-27T21:15:12,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742176_1352 (size=13391) 2024-11-27T21:15:12,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742177_1353 (size=13391) 2024-11-27T21:15:12,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742172313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:12,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:12,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742172315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:12,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:12,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 437 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742172316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:12,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742172318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:12,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:15:12,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:12,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:12,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:12,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T21:15:12,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742172429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:12,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742172429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:12,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 438 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742172429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742172429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:12,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:15:12,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:12,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:12,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:12,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:12,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:12,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742172639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:12,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:12,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 439 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742172639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742172639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:12,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742172640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:12,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=912 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/02853a04ddb64b6baaaa75448501ce5c 2024-11-27T21:15:12,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/288bfd82a7614c6280c3d5eb0cfda30c is 50, key is test_row_0/B:col10/1732742112234/Put/seqid=0 2024-11-27T21:15:12,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742178_1354 (size=12301) 2024-11-27T21:15:12,702 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/15334cb17f5c403a9baa1fe5884d1e85 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/15334cb17f5c403a9baa1fe5884d1e85 2024-11-27T21:15:12,702 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/320e5e6b7e094d76b079e0658e8bd0c1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/320e5e6b7e094d76b079e0658e8bd0c1 2024-11-27T21:15:12,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:15:12,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:12,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:12,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:12,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:12,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:12,708 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/A of acbb7f305d2f548aa38b4230ffde5235 into 15334cb17f5c403a9baa1fe5884d1e85(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:12,708 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:12,708 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/A, priority=12, startTime=1732742112267; duration=0sec 2024-11-27T21:15:12,708 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:12,708 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:A 2024-11-27T21:15:12,708 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:15:12,709 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/B of acbb7f305d2f548aa38b4230ffde5235 into 320e5e6b7e094d76b079e0658e8bd0c1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:12,709 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:12,709 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/B, priority=12, startTime=1732742112267; duration=0sec 2024-11-27T21:15:12,710 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:12,710 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:B 2024-11-27T21:15:12,710 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:15:12,710 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): acbb7f305d2f548aa38b4230ffde5235/C is initiating minor compaction (all files) 2024-11-27T21:15:12,710 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acbb7f305d2f548aa38b4230ffde5235/C in TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:12,710 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/91878d14210e48999a11f16f2c1e28f2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/434247733a284708aec9b2db9a23c458, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/a3208746fbb241c3b57e997832353482, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/13296fff43e04895a894b41929e162b2] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp, totalSize=48.9 K 2024-11-27T21:15:12,711 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91878d14210e48999a11f16f2c1e28f2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=853, earliestPutTs=1732742108967 2024-11-27T21:15:12,711 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 434247733a284708aec9b2db9a23c458, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=873, earliestPutTs=1732742110117 2024-11-27T21:15:12,711 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3208746fbb241c3b57e997832353482, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=888, earliestPutTs=1732742110996 2024-11-27T21:15:12,712 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13296fff43e04895a894b41929e162b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=898, earliestPutTs=1732742112195 2024-11-27T21:15:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T21:15:12,722 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acbb7f305d2f548aa38b4230ffde5235#C#compaction#305 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:12,723 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/8a4a0ac7e7d947fda1ae222fe2bac3b1 is 50, key is test_row_0/C:col10/1732742112201/Put/seqid=0 2024-11-27T21:15:12,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742179_1355 (size=13323) 2024-11-27T21:15:12,739 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/8a4a0ac7e7d947fda1ae222fe2bac3b1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/8a4a0ac7e7d947fda1ae222fe2bac3b1 2024-11-27T21:15:12,745 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in acbb7f305d2f548aa38b4230ffde5235/C of acbb7f305d2f548aa38b4230ffde5235 into 8a4a0ac7e7d947fda1ae222fe2bac3b1(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:12,745 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:12,745 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., storeName=acbb7f305d2f548aa38b4230ffde5235/C, priority=12, startTime=1732742112267; duration=0sec 2024-11-27T21:15:12,745 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:12,745 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acbb7f305d2f548aa38b4230ffde5235:C 2024-11-27T21:15:12,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:15:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:12,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:12,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:12,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742172949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:12,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 440 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742172949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:12,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742172949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:12,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742172950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:13,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:15:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:13,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:13,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:13,014 INFO [Thread-1201 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1201. 2024-11-27T21:15:13,014 DEBUG [Thread-1201 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:13,014 DEBUG [Thread-1201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:13,015 DEBUG [Thread-1201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:13,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:13,016 INFO [Thread-1203 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1203. 2024-11-27T21:15:13,016 DEBUG [Thread-1203 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:13,016 DEBUG [Thread-1203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:13,016 DEBUG [Thread-1203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:13,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:13,017 INFO [Thread-1199 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1199. 2024-11-27T21:15:13,017 DEBUG [Thread-1199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:13,017 DEBUG [Thread-1199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:13,017 DEBUG [Thread-1199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:13,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:13,019 INFO [Thread-1195 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1195. 2024-11-27T21:15:13,019 DEBUG [Thread-1195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:13,019 DEBUG [Thread-1195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:13,019 DEBUG [Thread-1195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:13,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:13,020 INFO [Thread-1197 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1197. 2024-11-27T21:15:13,020 DEBUG [Thread-1197 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:13,020 DEBUG [Thread-1197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:13,020 DEBUG [Thread-1197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:13,020 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:13,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=912 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/288bfd82a7614c6280c3d5eb0cfda30c 2024-11-27T21:15:13,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/97aab1560d1340b1af0a139d8ba3b34f is 50, key is test_row_0/C:col10/1732742112234/Put/seqid=0 2024-11-27T21:15:13,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742180_1356 (size=12301) 2024-11-27T21:15:13,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:15:13,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:13,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:13,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:13,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:13,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:13,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:13,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T21:15:13,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:15:13,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:13,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:13,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:13,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:13,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:13,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:13,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 441 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54022 deadline: 1732742173459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53992 deadline: 1732742173459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:13,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:13,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1732742173468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53970 deadline: 1732742173468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:13,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:15:13,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:13,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:13,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:13,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:13,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:13,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=912 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/97aab1560d1340b1af0a139d8ba3b34f 2024-11-27T21:15:13,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/02853a04ddb64b6baaaa75448501ce5c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/02853a04ddb64b6baaaa75448501ce5c 2024-11-27T21:15:13,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/02853a04ddb64b6baaaa75448501ce5c, entries=200, sequenceid=912, filesize=14.4 K 2024-11-27T21:15:13,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/288bfd82a7614c6280c3d5eb0cfda30c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/288bfd82a7614c6280c3d5eb0cfda30c 2024-11-27T21:15:13,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/288bfd82a7614c6280c3d5eb0cfda30c, entries=150, sequenceid=912, filesize=12.0 K 2024-11-27T21:15:13,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/97aab1560d1340b1af0a139d8ba3b34f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/97aab1560d1340b1af0a139d8ba3b34f 2024-11-27T21:15:13,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/97aab1560d1340b1af0a139d8ba3b34f, entries=150, sequenceid=912, filesize=12.0 K 2024-11-27T21:15:13,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for acbb7f305d2f548aa38b4230ffde5235 in 1248ms, sequenceid=912, compaction requested=false 2024-11-27T21:15:13,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:13,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T21:15:13,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:13,625 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-27T21:15:13,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:13,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:13,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:13,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:13,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:13,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:13,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/be1e92f88c0b423f9b3a56e55e8513f5 is 50, key is test_row_0/A:col10/1732742112314/Put/seqid=0 2024-11-27T21:15:13,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742181_1357 (size=12301) 2024-11-27T21:15:14,033 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=933 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/be1e92f88c0b423f9b3a56e55e8513f5 2024-11-27T21:15:14,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/1ef075d93db740b0b70bd03a01477ebf is 50, key is test_row_0/B:col10/1732742112314/Put/seqid=0 2024-11-27T21:15:14,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742182_1358 (size=12301) 2024-11-27T21:15:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T21:15:14,443 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=933 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/1ef075d93db740b0b70bd03a01477ebf 2024-11-27T21:15:14,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/2ebfb6155bbf43689c924e0f5137fa1b is 50, key is test_row_0/C:col10/1732742112314/Put/seqid=0 2024-11-27T21:15:14,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742183_1359 (size=12301) 2024-11-27T21:15:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:14,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. as already flushing 2024-11-27T21:15:14,470 INFO [Thread-1190 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1190. 2024-11-27T21:15:14,471 DEBUG [Thread-1190 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:14,471 DEBUG [Thread-1190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:14,471 DEBUG [Thread-1190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:14,471 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:14,480 INFO [Thread-1184 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1184. 2024-11-27T21:15:14,480 INFO [Thread-1192 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1192. 2024-11-27T21:15:14,480 DEBUG [Thread-1184 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:14,480 DEBUG [Thread-1192 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:14,480 DEBUG [Thread-1184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:14,480 DEBUG [Thread-1192 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:14,480 INFO [Thread-1186 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1186. 2024-11-27T21:15:14,480 DEBUG [Thread-1184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:14,480 DEBUG [Thread-1186 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:14,480 DEBUG [Thread-1192 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:14,480 DEBUG [Thread-1186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:14,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:14,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:14,480 DEBUG [Thread-1186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:14,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:14,853 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=933 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/2ebfb6155bbf43689c924e0f5137fa1b 2024-11-27T21:15:14,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/be1e92f88c0b423f9b3a56e55e8513f5 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/be1e92f88c0b423f9b3a56e55e8513f5 2024-11-27T21:15:14,860 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/be1e92f88c0b423f9b3a56e55e8513f5, entries=150, sequenceid=933, filesize=12.0 K 2024-11-27T21:15:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/1ef075d93db740b0b70bd03a01477ebf as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/1ef075d93db740b0b70bd03a01477ebf 2024-11-27T21:15:14,864 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/1ef075d93db740b0b70bd03a01477ebf, entries=150, sequenceid=933, filesize=12.0 K 2024-11-27T21:15:14,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/2ebfb6155bbf43689c924e0f5137fa1b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/2ebfb6155bbf43689c924e0f5137fa1b 2024-11-27T21:15:14,868 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/2ebfb6155bbf43689c924e0f5137fa1b, entries=150, sequenceid=933, filesize=12.0 K 2024-11-27T21:15:14,869 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=26.84 KB/27480 for acbb7f305d2f548aa38b4230ffde5235 in 1244ms, sequenceid=933, compaction requested=true 2024-11-27T21:15:14,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for acbb7f305d2f548aa38b4230ffde5235: 2024-11-27T21:15:14,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:14,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-27T21:15:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-27T21:15:14,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-27T21:15:14,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7770 sec 2024-11-27T21:15:14,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 2.7820 sec 2024-11-27T21:15:15,620 INFO [Thread-1188 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1188. 2024-11-27T21:15:15,621 DEBUG [Thread-1188 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:15,621 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:15,621 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:15,621 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T21:15:16,249 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:16,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T21:15:16,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-27T21:15:16,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 150 2024-11-27T21:15:16,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-11-27T21:15:16,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 384 2024-11-27T21:15:16,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-27T21:15:16,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T21:15:16,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2812 2024-11-27T21:15:16,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2774 2024-11-27T21:15:16,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2767 2024-11-27T21:15:16,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2829 2024-11-27T21:15:16,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2796 2024-11-27T21:15:16,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T21:15:16,250 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T21:15:16,250 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:15:16,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:16,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:16,250 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:16,251 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T21:15:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T21:15:16,254 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742116254"}]},"ts":"1732742116254"} 2024-11-27T21:15:16,255 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T21:15:16,255 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T21:15:16,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:15:16,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=acbb7f305d2f548aa38b4230ffde5235, UNASSIGN}] 2024-11-27T21:15:16,257 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=acbb7f305d2f548aa38b4230ffde5235, UNASSIGN 2024-11-27T21:15:16,258 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=acbb7f305d2f548aa38b4230ffde5235, regionState=CLOSING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:15:16,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=acbb7f305d2f548aa38b4230ffde5235, UNASSIGN because future has completed 2024-11-27T21:15:16,259 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:15:16,259 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:15:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T21:15:16,411 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:16,411 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:15:16,411 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing acbb7f305d2f548aa38b4230ffde5235, disabling compactions & flushes 2024-11-27T21:15:16,411 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:16,411 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:16,411 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. after waiting 0 ms 2024-11-27T21:15:16,412 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:16,412 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(2902): Flushing acbb7f305d2f548aa38b4230ffde5235 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-27T21:15:16,412 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=A 2024-11-27T21:15:16,412 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:16,412 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=B 2024-11-27T21:15:16,412 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:16,412 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.CompactingMemStore(205): FLUSHING TO DISK acbb7f305d2f548aa38b4230ffde5235, store=C 2024-11-27T21:15:16,412 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:16,415 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/dfddb399cf8c4cd08dac4801be803fe4 is 50, key is test_row_0/A:col10/1732742115619/Put/seqid=0 2024-11-27T21:15:16,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742184_1360 (size=12301) 2024-11-27T21:15:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T21:15:16,819 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=941 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/dfddb399cf8c4cd08dac4801be803fe4 2024-11-27T21:15:16,825 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/df4738976e7b4830b5e819f533b4ae81 is 50, key is test_row_0/B:col10/1732742115619/Put/seqid=0 2024-11-27T21:15:16,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742185_1361 (size=12301) 2024-11-27T21:15:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T21:15:17,229 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=941 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/df4738976e7b4830b5e819f533b4ae81 2024-11-27T21:15:17,236 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/cacefd6b9ed44387aca85ed835dd0de7 is 50, key is test_row_0/C:col10/1732742115619/Put/seqid=0 2024-11-27T21:15:17,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742186_1362 (size=12301) 2024-11-27T21:15:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T21:15:17,640 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=941 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/cacefd6b9ed44387aca85ed835dd0de7 2024-11-27T21:15:17,644 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/A/dfddb399cf8c4cd08dac4801be803fe4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/dfddb399cf8c4cd08dac4801be803fe4 2024-11-27T21:15:17,647 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/dfddb399cf8c4cd08dac4801be803fe4, entries=150, sequenceid=941, filesize=12.0 K 2024-11-27T21:15:17,648 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/B/df4738976e7b4830b5e819f533b4ae81 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/df4738976e7b4830b5e819f533b4ae81 2024-11-27T21:15:17,651 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/df4738976e7b4830b5e819f533b4ae81, entries=150, sequenceid=941, filesize=12.0 K 2024-11-27T21:15:17,651 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/.tmp/C/cacefd6b9ed44387aca85ed835dd0de7 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/cacefd6b9ed44387aca85ed835dd0de7 2024-11-27T21:15:17,654 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/cacefd6b9ed44387aca85ed835dd0de7, entries=150, sequenceid=941, filesize=12.0 K 2024-11-27T21:15:17,654 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for acbb7f305d2f548aa38b4230ffde5235 in 1242ms, sequenceid=941, compaction requested=true 2024-11-27T21:15:17,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/7cb024c46f0341d4a3fbbded7f00af53, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8f676558221a401b8f5648968b594257, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/de9f29a90458421e99c9e6753a310674, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/1b2cacea7bec4c30a1dbaa20f316381e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/5e47402b372f4a98a94cfc611ddada6f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/708922a58f814f76a90ed89b35b81c4b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/3260eefccf4e4bca9fd432a2832909ec, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/86bcf38b399244aba378c6afd0297ad1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/73bab60764f940428dbdb237246d0f82, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/a6f82987713444cea4933392eda4f1fd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/4b55fc51eeff474582ce6eb3ecaf8469, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/738ec392121a4744aac4aa8927e29462, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/21ff212559fe497fbeae2fceb0e08e72, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/1669f3a1ef8a473692cb42d0cfeee317, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/e9a6047cdb0741d099392d371f88bafb, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/6f95e1c1ed63498aaedac7154270a260, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8c3b621a8261421b8f58278b6f813e45, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/d68acaf1cf874ea0bc2e880cae1d4f8c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/100585a8335c4e7f8015c337ca0f9d74, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/4929423bb1944f66834bc6e0bab1b203, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/018fe94e21454f1db6b9ebd1b2a1f1a7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/ddd69e795e564abdafd115504b6d3006, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8903513aa1ce411e8c68fb557e3ca43d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/53fb8b16ece14da681cc45a5985a3481, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/df7549a162144001babd4a5abf14889d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/622fecec151147b9831499b3c8e320cb, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/c549e4bb36d24842bf3da228d05705f8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/37e687cb5b5d408fa7d3ddd2279bf3e1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/3d2e84f8ce96413ea0aeb2259dbc1041, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/0aeea06bbe584522a3c784b5afea022e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/aae1424948af4c698ddb8e83ed9231dd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/e86065a5b7ab4a55b522b781f9308aa4] to archive 2024-11-27T21:15:17,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:15:17,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/7cb024c46f0341d4a3fbbded7f00af53 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/7cb024c46f0341d4a3fbbded7f00af53 2024-11-27T21:15:17,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8f676558221a401b8f5648968b594257 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8f676558221a401b8f5648968b594257 2024-11-27T21:15:17,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/de9f29a90458421e99c9e6753a310674 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/de9f29a90458421e99c9e6753a310674 2024-11-27T21:15:17,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/1b2cacea7bec4c30a1dbaa20f316381e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/1b2cacea7bec4c30a1dbaa20f316381e 2024-11-27T21:15:17,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/5e47402b372f4a98a94cfc611ddada6f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/5e47402b372f4a98a94cfc611ddada6f 2024-11-27T21:15:17,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/708922a58f814f76a90ed89b35b81c4b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/708922a58f814f76a90ed89b35b81c4b 2024-11-27T21:15:17,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/3260eefccf4e4bca9fd432a2832909ec to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/3260eefccf4e4bca9fd432a2832909ec 2024-11-27T21:15:17,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/86bcf38b399244aba378c6afd0297ad1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/86bcf38b399244aba378c6afd0297ad1 2024-11-27T21:15:17,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/73bab60764f940428dbdb237246d0f82 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/73bab60764f940428dbdb237246d0f82 2024-11-27T21:15:17,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/a6f82987713444cea4933392eda4f1fd to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/a6f82987713444cea4933392eda4f1fd 2024-11-27T21:15:17,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/4b55fc51eeff474582ce6eb3ecaf8469 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/4b55fc51eeff474582ce6eb3ecaf8469 2024-11-27T21:15:17,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/738ec392121a4744aac4aa8927e29462 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/738ec392121a4744aac4aa8927e29462 2024-11-27T21:15:17,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/21ff212559fe497fbeae2fceb0e08e72 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/21ff212559fe497fbeae2fceb0e08e72 2024-11-27T21:15:17,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/1669f3a1ef8a473692cb42d0cfeee317 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/1669f3a1ef8a473692cb42d0cfeee317 2024-11-27T21:15:17,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/e9a6047cdb0741d099392d371f88bafb to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/e9a6047cdb0741d099392d371f88bafb 2024-11-27T21:15:17,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/6f95e1c1ed63498aaedac7154270a260 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/6f95e1c1ed63498aaedac7154270a260 2024-11-27T21:15:17,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8c3b621a8261421b8f58278b6f813e45 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8c3b621a8261421b8f58278b6f813e45 2024-11-27T21:15:17,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/d68acaf1cf874ea0bc2e880cae1d4f8c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/d68acaf1cf874ea0bc2e880cae1d4f8c 2024-11-27T21:15:17,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/100585a8335c4e7f8015c337ca0f9d74 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/100585a8335c4e7f8015c337ca0f9d74 2024-11-27T21:15:17,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/4929423bb1944f66834bc6e0bab1b203 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/4929423bb1944f66834bc6e0bab1b203 2024-11-27T21:15:17,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/018fe94e21454f1db6b9ebd1b2a1f1a7 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/018fe94e21454f1db6b9ebd1b2a1f1a7 2024-11-27T21:15:17,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/ddd69e795e564abdafd115504b6d3006 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/ddd69e795e564abdafd115504b6d3006 2024-11-27T21:15:17,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8903513aa1ce411e8c68fb557e3ca43d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/8903513aa1ce411e8c68fb557e3ca43d 2024-11-27T21:15:17,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/53fb8b16ece14da681cc45a5985a3481 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/53fb8b16ece14da681cc45a5985a3481 2024-11-27T21:15:17,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/df7549a162144001babd4a5abf14889d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/df7549a162144001babd4a5abf14889d 2024-11-27T21:15:17,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/622fecec151147b9831499b3c8e320cb to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/622fecec151147b9831499b3c8e320cb 2024-11-27T21:15:17,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/c549e4bb36d24842bf3da228d05705f8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/c549e4bb36d24842bf3da228d05705f8 2024-11-27T21:15:17,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/37e687cb5b5d408fa7d3ddd2279bf3e1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/37e687cb5b5d408fa7d3ddd2279bf3e1 2024-11-27T21:15:17,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/3d2e84f8ce96413ea0aeb2259dbc1041 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/3d2e84f8ce96413ea0aeb2259dbc1041 2024-11-27T21:15:17,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/0aeea06bbe584522a3c784b5afea022e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/0aeea06bbe584522a3c784b5afea022e 2024-11-27T21:15:17,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/aae1424948af4c698ddb8e83ed9231dd to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/aae1424948af4c698ddb8e83ed9231dd 2024-11-27T21:15:17,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/e86065a5b7ab4a55b522b781f9308aa4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/e86065a5b7ab4a55b522b781f9308aa4 2024-11-27T21:15:17,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/098053f7982d47c183d8bc838cc984c0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/55968037277b4f35b2d77df8f9871a75, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/fa01dcb60db449479008b906dc50e6ae, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/feb7d215e90d4cae8f5431a77a37a01d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/305a0aac471446d890cbd0183738c123, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/3b6fa1e0eae64f3ab1eac90ff6a74e2e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/09601a227931429f9db15695535a6090, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/e032da030eca496886b3a0f00febb5ff, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/51ff4fade3f946e99177b408fc31c362, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/12d2a2b8199a478fbe670c75320e7b83, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/e38fdd5f04f9417f91d5df829ac2604b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/2f2bf22489444ec89cdfa717a87c3d50, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/69738dfa28494aa29ca03969d18f0fef, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/88d25563360542a2ac76a3cfbce7d0d2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/34821fedb5bf43718dccda40a757da6e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/b9093b2f18804df399736f038b602ac7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/6ef55cda8c2e4ab3a4c672b11d32f053, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/6150742aecef400aacd73273d4435cee, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/532f713802074883b0d2adc1f54c4608, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/20ff4b28ffec4df1ac9e94bdbe67433e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/0ec5a08cb7094429bad393dfda63bea4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/5e60a7ee8b5149d198b400c5da25c9ef, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/9cf0a4b48abd4c2f9b82582cef95f562, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/a991de8dbb1e40fe8f9bb57e552ed0c6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/f44394a4b25b434ea74f3de91cc8e6d6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/b3ca71a87e7741e7b8ccdcc77086dab8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/00ec20d8af4d4f91b09051ab4412a0f0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/5aacd33604b142b69e7679633e6b184a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/80d5d2504d9a44ebb23d81df5d79ff57, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/88126383a4bd4af69baa1d796daa6fea, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/1b7bf6b09fc64684a5ee2e65ef9ad560, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/ecda8cff631949638b7997897d1f2575] to archive 2024-11-27T21:15:17,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:15:17,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/098053f7982d47c183d8bc838cc984c0 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/098053f7982d47c183d8bc838cc984c0 2024-11-27T21:15:17,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/55968037277b4f35b2d77df8f9871a75 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/55968037277b4f35b2d77df8f9871a75 2024-11-27T21:15:17,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/fa01dcb60db449479008b906dc50e6ae to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/fa01dcb60db449479008b906dc50e6ae 2024-11-27T21:15:17,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/feb7d215e90d4cae8f5431a77a37a01d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/feb7d215e90d4cae8f5431a77a37a01d 2024-11-27T21:15:17,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/305a0aac471446d890cbd0183738c123 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/305a0aac471446d890cbd0183738c123 2024-11-27T21:15:17,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/3b6fa1e0eae64f3ab1eac90ff6a74e2e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/3b6fa1e0eae64f3ab1eac90ff6a74e2e 2024-11-27T21:15:17,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/09601a227931429f9db15695535a6090 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/09601a227931429f9db15695535a6090 2024-11-27T21:15:17,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/e032da030eca496886b3a0f00febb5ff to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/e032da030eca496886b3a0f00febb5ff 2024-11-27T21:15:17,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/51ff4fade3f946e99177b408fc31c362 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/51ff4fade3f946e99177b408fc31c362 2024-11-27T21:15:17,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/12d2a2b8199a478fbe670c75320e7b83 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/12d2a2b8199a478fbe670c75320e7b83 2024-11-27T21:15:17,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/e38fdd5f04f9417f91d5df829ac2604b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/e38fdd5f04f9417f91d5df829ac2604b 2024-11-27T21:15:17,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/2f2bf22489444ec89cdfa717a87c3d50 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/2f2bf22489444ec89cdfa717a87c3d50 2024-11-27T21:15:17,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/69738dfa28494aa29ca03969d18f0fef to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/69738dfa28494aa29ca03969d18f0fef 2024-11-27T21:15:17,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/88d25563360542a2ac76a3cfbce7d0d2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/88d25563360542a2ac76a3cfbce7d0d2 2024-11-27T21:15:17,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/34821fedb5bf43718dccda40a757da6e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/34821fedb5bf43718dccda40a757da6e 2024-11-27T21:15:17,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/b9093b2f18804df399736f038b602ac7 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/b9093b2f18804df399736f038b602ac7 2024-11-27T21:15:17,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/6ef55cda8c2e4ab3a4c672b11d32f053 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/6ef55cda8c2e4ab3a4c672b11d32f053 2024-11-27T21:15:17,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/6150742aecef400aacd73273d4435cee to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/6150742aecef400aacd73273d4435cee 2024-11-27T21:15:17,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/532f713802074883b0d2adc1f54c4608 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/532f713802074883b0d2adc1f54c4608 2024-11-27T21:15:17,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/20ff4b28ffec4df1ac9e94bdbe67433e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/20ff4b28ffec4df1ac9e94bdbe67433e 2024-11-27T21:15:17,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/0ec5a08cb7094429bad393dfda63bea4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/0ec5a08cb7094429bad393dfda63bea4 2024-11-27T21:15:17,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/5e60a7ee8b5149d198b400c5da25c9ef to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/5e60a7ee8b5149d198b400c5da25c9ef 2024-11-27T21:15:17,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/9cf0a4b48abd4c2f9b82582cef95f562 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/9cf0a4b48abd4c2f9b82582cef95f562 2024-11-27T21:15:17,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/a991de8dbb1e40fe8f9bb57e552ed0c6 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/a991de8dbb1e40fe8f9bb57e552ed0c6 2024-11-27T21:15:17,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/f44394a4b25b434ea74f3de91cc8e6d6 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/f44394a4b25b434ea74f3de91cc8e6d6 2024-11-27T21:15:17,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/b3ca71a87e7741e7b8ccdcc77086dab8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/b3ca71a87e7741e7b8ccdcc77086dab8 2024-11-27T21:15:17,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/00ec20d8af4d4f91b09051ab4412a0f0 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/00ec20d8af4d4f91b09051ab4412a0f0 2024-11-27T21:15:17,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/5aacd33604b142b69e7679633e6b184a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/5aacd33604b142b69e7679633e6b184a 2024-11-27T21:15:17,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/80d5d2504d9a44ebb23d81df5d79ff57 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/80d5d2504d9a44ebb23d81df5d79ff57 2024-11-27T21:15:17,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/88126383a4bd4af69baa1d796daa6fea to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/88126383a4bd4af69baa1d796daa6fea 2024-11-27T21:15:17,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/1b7bf6b09fc64684a5ee2e65ef9ad560 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/1b7bf6b09fc64684a5ee2e65ef9ad560 2024-11-27T21:15:17,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/ecda8cff631949638b7997897d1f2575 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/ecda8cff631949638b7997897d1f2575 2024-11-27T21:15:17,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/dd7bceeb9ca442f6b5831d7161d1291f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/74a17ecd1c11459dbb65f9f5d1dadeb6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/53b4f398bd0740e6aa3ab7b13ed0a3cd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/91cb3b909c504b258d1db94c67b0f467, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c4a9c4449b21427bbc27dbe4b13d9abe, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/d6f4871f5e374c3ea715b998af2cd40f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/5df4254c12cd497c809708208eabc012, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c1262c8893c74a8aaee53dec437df310, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/f218bc62967f45beae00d58075d5c962, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/626ccbc976154dd58f969d784785c9e1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b08e6954aed24141843962458ec30f90, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/eeeac25d45c54af281741f0ba9df0ae9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c515bb5eac7847769a689a257b29755a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c80acf6950294716ad03ace62af91d7d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b5f872fbfa6d4b648cc3c23941028a16, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/3ca52e60831e4030adbd0ce07c5fcd69, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/3def572d21474bb2b77d0445bf9d0636, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/6259df03d2a149ceb4166612b336783d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/a942f96da6404fc1a888086490637069, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/f7b0d19133ff41b8b1bf7122520edfc0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/6336a7bfa23e4f47bf7317b431ddb77f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/e02c8cf2f03a4144a3da464d0633ab14, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/9666dfec0a064229ab8aa99835fcec98, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c30fad3e4cf74c589290bd7242b31963, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b33bc6d65cac49c1a3c3e53466fe981d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/91878d14210e48999a11f16f2c1e28f2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/44ca63e6c25142a7a539ffab520db0de, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/434247733a284708aec9b2db9a23c458, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/a3208746fbb241c3b57e997832353482, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/13296fff43e04895a894b41929e162b2] to archive 2024-11-27T21:15:17,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:15:17,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/dd7bceeb9ca442f6b5831d7161d1291f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/dd7bceeb9ca442f6b5831d7161d1291f 2024-11-27T21:15:17,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/74a17ecd1c11459dbb65f9f5d1dadeb6 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/74a17ecd1c11459dbb65f9f5d1dadeb6 2024-11-27T21:15:17,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/53b4f398bd0740e6aa3ab7b13ed0a3cd to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/53b4f398bd0740e6aa3ab7b13ed0a3cd 2024-11-27T21:15:17,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/91cb3b909c504b258d1db94c67b0f467 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/91cb3b909c504b258d1db94c67b0f467 2024-11-27T21:15:17,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c4a9c4449b21427bbc27dbe4b13d9abe to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c4a9c4449b21427bbc27dbe4b13d9abe 2024-11-27T21:15:17,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/d6f4871f5e374c3ea715b998af2cd40f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/d6f4871f5e374c3ea715b998af2cd40f 2024-11-27T21:15:17,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/5df4254c12cd497c809708208eabc012 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/5df4254c12cd497c809708208eabc012 2024-11-27T21:15:17,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c1262c8893c74a8aaee53dec437df310 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c1262c8893c74a8aaee53dec437df310 2024-11-27T21:15:17,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/f218bc62967f45beae00d58075d5c962 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/f218bc62967f45beae00d58075d5c962 2024-11-27T21:15:17,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/626ccbc976154dd58f969d784785c9e1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/626ccbc976154dd58f969d784785c9e1 2024-11-27T21:15:17,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b08e6954aed24141843962458ec30f90 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b08e6954aed24141843962458ec30f90 2024-11-27T21:15:17,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/eeeac25d45c54af281741f0ba9df0ae9 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/eeeac25d45c54af281741f0ba9df0ae9 2024-11-27T21:15:17,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c515bb5eac7847769a689a257b29755a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c515bb5eac7847769a689a257b29755a 2024-11-27T21:15:17,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c80acf6950294716ad03ace62af91d7d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c80acf6950294716ad03ace62af91d7d 2024-11-27T21:15:17,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b5f872fbfa6d4b648cc3c23941028a16 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b5f872fbfa6d4b648cc3c23941028a16 2024-11-27T21:15:17,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/3ca52e60831e4030adbd0ce07c5fcd69 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/3ca52e60831e4030adbd0ce07c5fcd69 2024-11-27T21:15:17,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/3def572d21474bb2b77d0445bf9d0636 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/3def572d21474bb2b77d0445bf9d0636 2024-11-27T21:15:17,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/6259df03d2a149ceb4166612b336783d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/6259df03d2a149ceb4166612b336783d 2024-11-27T21:15:17,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/a942f96da6404fc1a888086490637069 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/a942f96da6404fc1a888086490637069 2024-11-27T21:15:17,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/f7b0d19133ff41b8b1bf7122520edfc0 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/f7b0d19133ff41b8b1bf7122520edfc0 2024-11-27T21:15:17,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/6336a7bfa23e4f47bf7317b431ddb77f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/6336a7bfa23e4f47bf7317b431ddb77f 2024-11-27T21:15:17,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/e02c8cf2f03a4144a3da464d0633ab14 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/e02c8cf2f03a4144a3da464d0633ab14 2024-11-27T21:15:17,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/9666dfec0a064229ab8aa99835fcec98 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/9666dfec0a064229ab8aa99835fcec98 2024-11-27T21:15:17,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c30fad3e4cf74c589290bd7242b31963 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/c30fad3e4cf74c589290bd7242b31963 2024-11-27T21:15:17,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b33bc6d65cac49c1a3c3e53466fe981d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/b33bc6d65cac49c1a3c3e53466fe981d 2024-11-27T21:15:17,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/91878d14210e48999a11f16f2c1e28f2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/91878d14210e48999a11f16f2c1e28f2 2024-11-27T21:15:17,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/44ca63e6c25142a7a539ffab520db0de to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/44ca63e6c25142a7a539ffab520db0de 2024-11-27T21:15:17,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/434247733a284708aec9b2db9a23c458 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/434247733a284708aec9b2db9a23c458 2024-11-27T21:15:17,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/a3208746fbb241c3b57e997832353482 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/a3208746fbb241c3b57e997832353482 2024-11-27T21:15:17,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/13296fff43e04895a894b41929e162b2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/13296fff43e04895a894b41929e162b2 2024-11-27T21:15:17,755 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/recovered.edits/944.seqid, newMaxSeqId=944, maxSeqId=1 2024-11-27T21:15:17,755 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. 2024-11-27T21:15:17,755 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for acbb7f305d2f548aa38b4230ffde5235: Waiting for close lock at 1732742116411Running coprocessor pre-close hooks at 1732742116411Disabling compacts and flushes for region at 1732742116411Disabling writes for close at 1732742116411Obtaining lock to block concurrent updates at 1732742116412 (+1 ms)Preparing flush snapshotting stores in acbb7f305d2f548aa38b4230ffde5235 at 1732742116412Finished memstore snapshotting TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732742116412Flushing stores of TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235. at 1732742116412Flushing acbb7f305d2f548aa38b4230ffde5235/A: creating writer at 1732742116413 (+1 ms)Flushing acbb7f305d2f548aa38b4230ffde5235/A: appending metadata at 1732742116415 (+2 ms)Flushing acbb7f305d2f548aa38b4230ffde5235/A: closing flushed file at 1732742116415Flushing acbb7f305d2f548aa38b4230ffde5235/B: creating writer at 1732742116823 (+408 ms)Flushing acbb7f305d2f548aa38b4230ffde5235/B: appending metadata at 1732742116825 (+2 ms)Flushing acbb7f305d2f548aa38b4230ffde5235/B: closing flushed file at 1732742116825Flushing acbb7f305d2f548aa38b4230ffde5235/C: creating writer at 1732742117233 (+408 ms)Flushing acbb7f305d2f548aa38b4230ffde5235/C: appending metadata at 1732742117236 (+3 ms)Flushing acbb7f305d2f548aa38b4230ffde5235/C: closing flushed file at 1732742117236Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@142d6b28: reopening flushed file at 1732742117644 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ddf5516: reopening flushed file at 1732742117647 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ea81132: reopening flushed file at 1732742117651 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for acbb7f305d2f548aa38b4230ffde5235 in 1242ms, sequenceid=941, compaction requested=true at 1732742117654 (+3 ms)Writing region close event to WAL at 1732742117752 (+98 ms)Running coprocessor post-close hooks at 1732742117755 (+3 ms)Closed at 1732742117755 2024-11-27T21:15:17,757 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:17,758 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=acbb7f305d2f548aa38b4230ffde5235, regionState=CLOSED 2024-11-27T21:15:17,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:15:17,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-27T21:15:17,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure acbb7f305d2f548aa38b4230ffde5235, server=4585c39bed03,37381,1732742022265 in 1.5010 sec 2024-11-27T21:15:17,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-27T21:15:17,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=acbb7f305d2f548aa38b4230ffde5235, UNASSIGN in 1.5040 sec 2024-11-27T21:15:17,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-27T21:15:17,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5070 sec 2024-11-27T21:15:17,765 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742117765"}]},"ts":"1732742117765"} 2024-11-27T21:15:17,766 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T21:15:17,766 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T21:15:17,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5150 sec 2024-11-27T21:15:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T21:15:18,400 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:18,400 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T21:15:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:18,402 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-27T21:15:18,402 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=94, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:18,404 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:18,406 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/recovered.edits] 2024-11-27T21:15:18,408 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/02853a04ddb64b6baaaa75448501ce5c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/02853a04ddb64b6baaaa75448501ce5c 2024-11-27T21:15:18,409 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/15334cb17f5c403a9baa1fe5884d1e85 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/15334cb17f5c403a9baa1fe5884d1e85 2024-11-27T21:15:18,410 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/be1e92f88c0b423f9b3a56e55e8513f5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/be1e92f88c0b423f9b3a56e55e8513f5 2024-11-27T21:15:18,411 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/dfddb399cf8c4cd08dac4801be803fe4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/A/dfddb399cf8c4cd08dac4801be803fe4 2024-11-27T21:15:18,413 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/1ef075d93db740b0b70bd03a01477ebf to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/1ef075d93db740b0b70bd03a01477ebf 2024-11-27T21:15:18,414 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/288bfd82a7614c6280c3d5eb0cfda30c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/288bfd82a7614c6280c3d5eb0cfda30c 2024-11-27T21:15:18,414 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/320e5e6b7e094d76b079e0658e8bd0c1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/320e5e6b7e094d76b079e0658e8bd0c1 2024-11-27T21:15:18,416 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/df4738976e7b4830b5e819f533b4ae81 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/B/df4738976e7b4830b5e819f533b4ae81 2024-11-27T21:15:18,417 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/2ebfb6155bbf43689c924e0f5137fa1b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/2ebfb6155bbf43689c924e0f5137fa1b 2024-11-27T21:15:18,418 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/8a4a0ac7e7d947fda1ae222fe2bac3b1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/8a4a0ac7e7d947fda1ae222fe2bac3b1 2024-11-27T21:15:18,419 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/97aab1560d1340b1af0a139d8ba3b34f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/97aab1560d1340b1af0a139d8ba3b34f 2024-11-27T21:15:18,420 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/cacefd6b9ed44387aca85ed835dd0de7 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/C/cacefd6b9ed44387aca85ed835dd0de7 2024-11-27T21:15:18,422 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/recovered.edits/944.seqid to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235/recovered.edits/944.seqid 2024-11-27T21:15:18,423 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/acbb7f305d2f548aa38b4230ffde5235 2024-11-27T21:15:18,423 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T21:15:18,425 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=94, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:18,427 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T21:15:18,428 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T21:15:18,429 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=94, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:18,429 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T21:15:18,430 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732742118430"}]},"ts":"9223372036854775807"} 2024-11-27T21:15:18,431 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T21:15:18,431 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => acbb7f305d2f548aa38b4230ffde5235, NAME => 'TestAcidGuarantees,,1732742090818.acbb7f305d2f548aa38b4230ffde5235.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T21:15:18,431 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T21:15:18,432 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732742118431"}]},"ts":"9223372036854775807"} 2024-11-27T21:15:18,433 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T21:15:18,434 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=94, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:18,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-27T21:15:18,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-27T21:15:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T21:15:18,510 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:18,519 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_797614105_22 at /127.0.0.1:37284 [Waiting for operation #242] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1719228232_22 at /127.0.0.1:46390 [Waiting for operation #416] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=424 (was 465), ProcessCount=11 (was 11), AvailableMemoryMB=8519 (was 8720) 2024-11-27T21:15:18,527 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=424, ProcessCount=11, AvailableMemoryMB=8519 2024-11-27T21:15:18,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:15:18,529 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:15:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:18,531 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T21:15:18,531 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:18,531 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 95 2024-11-27T21:15:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T21:15:18,532 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T21:15:18,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742187_1363 (size=1061) 2024-11-27T21:15:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T21:15:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T21:15:18,938 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de 2024-11-27T21:15:18,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742188_1364 (size=53) 2024-11-27T21:15:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T21:15:19,344 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:15:19,344 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 49ff3edd8f78b405b2b26bdd2aa6e347, disabling compactions & flushes 2024-11-27T21:15:19,344 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:19,344 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:19,344 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. after waiting 0 ms 2024-11-27T21:15:19,344 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:19,344 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:19,344 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 49ff3edd8f78b405b2b26bdd2aa6e347: Waiting for close lock at 1732742119344Disabling compacts and flushes for region at 1732742119344Disabling writes for close at 1732742119344Writing region close event to WAL at 1732742119344Closed at 1732742119344 2024-11-27T21:15:19,345 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T21:15:19,345 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732742119345"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732742119345"}]},"ts":"1732742119345"} 2024-11-27T21:15:19,347 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T21:15:19,348 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T21:15:19,348 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742119348"}]},"ts":"1732742119348"} 2024-11-27T21:15:19,349 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T21:15:19,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49ff3edd8f78b405b2b26bdd2aa6e347, ASSIGN}] 2024-11-27T21:15:19,350 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49ff3edd8f78b405b2b26bdd2aa6e347, ASSIGN 2024-11-27T21:15:19,351 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49ff3edd8f78b405b2b26bdd2aa6e347, ASSIGN; state=OFFLINE, location=4585c39bed03,37381,1732742022265; forceNewPlan=false, retain=false 2024-11-27T21:15:19,502 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=49ff3edd8f78b405b2b26bdd2aa6e347, regionState=OPENING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:15:19,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49ff3edd8f78b405b2b26bdd2aa6e347, ASSIGN because future has completed 2024-11-27T21:15:19,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:15:19,659 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:19,660 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7752): Opening region: {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:15:19,660 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:19,660 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:15:19,660 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7794): checking encryption for 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:19,660 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7797): checking classloading for 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:19,661 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:19,662 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:15:19,663 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49ff3edd8f78b405b2b26bdd2aa6e347 columnFamilyName A 2024-11-27T21:15:19,663 DEBUG [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:19,663 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.HStore(327): Store=49ff3edd8f78b405b2b26bdd2aa6e347/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:15:19,663 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:19,664 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:15:19,664 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49ff3edd8f78b405b2b26bdd2aa6e347 columnFamilyName B 2024-11-27T21:15:19,664 DEBUG [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:19,665 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.HStore(327): Store=49ff3edd8f78b405b2b26bdd2aa6e347/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:15:19,665 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:19,666 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:15:19,666 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49ff3edd8f78b405b2b26bdd2aa6e347 columnFamilyName C 2024-11-27T21:15:19,666 DEBUG [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:19,666 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.HStore(327): Store=49ff3edd8f78b405b2b26bdd2aa6e347/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:15:19,666 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:19,666 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1038): replaying wal for 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:19,667 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:19,667 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:19,668 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1048): stopping wal replay for 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:19,668 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1060): Cleaning up temporary data for 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:19,669 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:15:19,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T21:15:19,670 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1093): writing seq id for 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:19,671 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T21:15:19,672 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1114): Opened 49ff3edd8f78b405b2b26bdd2aa6e347; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70820531, jitterRate=0.05530814826488495}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:15:19,672 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:19,673 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1006): Region open journal for 49ff3edd8f78b405b2b26bdd2aa6e347: Running coprocessor pre-open hook at 1732742119660Writing region info on filesystem at 1732742119660Initializing all the Stores at 1732742119661 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742119661Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742119661Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742119661Cleaning up temporary data from old regions at 1732742119668 (+7 ms)Running coprocessor post-open hooks at 1732742119672 (+4 ms)Region opened successfully at 1732742119673 (+1 ms) 2024-11-27T21:15:19,673 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., pid=97, masterSystemTime=1732742119656 2024-11-27T21:15:19,675 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:19,675 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:19,676 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=49ff3edd8f78b405b2b26bdd2aa6e347, regionState=OPEN, openSeqNum=2, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:15:19,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:15:19,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-27T21:15:19,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 in 173 msec 2024-11-27T21:15:19,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-27T21:15:19,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49ff3edd8f78b405b2b26bdd2aa6e347, ASSIGN in 331 msec 2024-11-27T21:15:19,682 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T21:15:19,683 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742119682"}]},"ts":"1732742119682"} 2024-11-27T21:15:19,684 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T21:15:19,684 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T21:15:19,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1550 sec 2024-11-27T21:15:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T21:15:20,679 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:20,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db9998b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:20,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:20,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:20,683 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:20,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:20,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:20,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20a2c416, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:20,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:20,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:20,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:20,685 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bf31d8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:20,686 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:20,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:20,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:20,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:20,690 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T21:15:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 4585c39bed03,38503,1732742021440 2024-11-27T21:15:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6da04309 2024-11-27T21:15:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T21:15:20,692 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T21:15:20,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:15:20,694 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:15:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:20,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742189_1365 (size=1097) 2024-11-27T21:15:21,115 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-27T21:15:21,115 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-27T21:15:21,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:15:21,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49ff3edd8f78b405b2b26bdd2aa6e347, REOPEN/MOVE}] 2024-11-27T21:15:21,120 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49ff3edd8f78b405b2b26bdd2aa6e347, REOPEN/MOVE 2024-11-27T21:15:21,121 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=49ff3edd8f78b405b2b26bdd2aa6e347, regionState=CLOSING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:15:21,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49ff3edd8f78b405b2b26bdd2aa6e347, REOPEN/MOVE because future has completed 2024-11-27T21:15:21,123 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:15:21,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure 49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:15:21,275 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(122): Close 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,275 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:15:21,276 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1722): Closing 49ff3edd8f78b405b2b26bdd2aa6e347, disabling compactions & flushes 2024-11-27T21:15:21,276 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:21,276 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:21,276 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. after waiting 0 ms 2024-11-27T21:15:21,276 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:21,281 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T21:15:21,281 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:21,281 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1676): Region close journal for 49ff3edd8f78b405b2b26bdd2aa6e347: Waiting for close lock at 1732742121276Running coprocessor pre-close hooks at 1732742121276Disabling compacts and flushes for region at 1732742121276Disabling writes for close at 1732742121276Writing region close event to WAL at 1732742121276Running coprocessor post-close hooks at 1732742121281 (+5 ms)Closed at 1732742121281 2024-11-27T21:15:21,282 WARN [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegionServer(3299): Not adding moved region record: 49ff3edd8f78b405b2b26bdd2aa6e347 to self. 2024-11-27T21:15:21,284 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(157): Closed 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,284 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=49ff3edd8f78b405b2b26bdd2aa6e347, regionState=CLOSED 2024-11-27T21:15:21,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure 49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:15:21,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-27T21:15:21,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; CloseRegionProcedure 49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 in 164 msec 2024-11-27T21:15:21,289 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49ff3edd8f78b405b2b26bdd2aa6e347, REOPEN/MOVE; state=CLOSED, location=4585c39bed03,37381,1732742022265; forceNewPlan=false, retain=true 2024-11-27T21:15:21,440 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=49ff3edd8f78b405b2b26bdd2aa6e347, regionState=OPENING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:15:21,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49ff3edd8f78b405b2b26bdd2aa6e347, REOPEN/MOVE because future has completed 2024-11-27T21:15:21,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:15:21,596 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:21,597 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7752): Opening region: {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:15:21,597 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,597 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:15:21,597 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7794): checking encryption for 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,597 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7797): checking classloading for 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,598 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,599 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:15:21,599 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49ff3edd8f78b405b2b26bdd2aa6e347 columnFamilyName A 2024-11-27T21:15:21,600 DEBUG [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:21,600 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.HStore(327): Store=49ff3edd8f78b405b2b26bdd2aa6e347/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:15:21,601 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,601 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:15:21,601 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49ff3edd8f78b405b2b26bdd2aa6e347 columnFamilyName B 2024-11-27T21:15:21,601 DEBUG [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:21,601 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.HStore(327): Store=49ff3edd8f78b405b2b26bdd2aa6e347/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:15:21,602 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,602 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:15:21,602 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49ff3edd8f78b405b2b26bdd2aa6e347 columnFamilyName C 2024-11-27T21:15:21,602 DEBUG [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:21,603 INFO [StoreOpener-49ff3edd8f78b405b2b26bdd2aa6e347-1 {}] regionserver.HStore(327): Store=49ff3edd8f78b405b2b26bdd2aa6e347/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:15:21,603 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:21,603 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1038): replaying wal for 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,603 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,604 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,605 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1048): stopping wal replay for 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,605 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1060): Cleaning up temporary data for 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,605 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:15:21,606 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1093): writing seq id for 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,607 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1114): Opened 49ff3edd8f78b405b2b26bdd2aa6e347; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66122936, jitterRate=-0.014691472053527832}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:15:21,607 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,607 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1006): Region open journal for 49ff3edd8f78b405b2b26bdd2aa6e347: Running coprocessor pre-open hook at 1732742121597Writing region info on filesystem at 1732742121597Initializing all the Stores at 1732742121598 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742121598Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742121598Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742121598Cleaning up temporary data from old regions at 1732742121605 (+7 ms)Running coprocessor post-open hooks at 1732742121607 (+2 ms)Region opened successfully at 1732742121607 2024-11-27T21:15:21,608 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., pid=102, masterSystemTime=1732742121594 2024-11-27T21:15:21,609 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:21,609 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:21,610 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=49ff3edd8f78b405b2b26bdd2aa6e347, regionState=OPEN, openSeqNum=5, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:15:21,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:15:21,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=100 2024-11-27T21:15:21,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=100, state=SUCCESS, hasLock=false; OpenRegionProcedure 49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 in 169 msec 2024-11-27T21:15:21,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-27T21:15:21,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49ff3edd8f78b405b2b26bdd2aa6e347, REOPEN/MOVE in 494 msec 2024-11-27T21:15:21,616 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-27T21:15:21,616 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 497 msec 2024-11-27T21:15:21,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 923 msec 2024-11-27T21:15:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-27T21:15:21,618 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:21,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:21,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:21,620 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25886168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:21,622 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:21,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29dcf54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,623 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:21,623 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:21,623 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:21,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a02f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:21,624 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4d2a0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,626 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:21,626 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:21,626 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:21,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:21,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:21,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a2846af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:21,627 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b9e10b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,628 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:21,628 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:21,629 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38a1c109, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:21,630 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae82aa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,631 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:21,631 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:21,631 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5420cb24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:21,633 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:21,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a17680e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:21,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:21,634 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:21,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:21,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:21,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@449061f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:21,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:21,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:21,635 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d48f07c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,636 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:21,636 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:21,637 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7402799b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:21,638 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7878e9c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:21,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:21,639 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26f538ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:21,641 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76a21650, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:21,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:21,642 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@323a5263, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:21,643 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:21,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@747202af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,644 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:21,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:21,645 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ccbdf24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:21,646 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d71458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:21,648 DEBUG [Thread-1661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:21,648 DEBUG [Thread-1667 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:21,649 DEBUG [Thread-1669 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:21,649 DEBUG [Thread-1652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:21,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:21,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:21,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:21,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:21,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:21,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:21,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:15:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:15:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:15:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:15:21,655 DEBUG [Thread-1665 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:21,656 DEBUG [Thread-1663 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:21,657 DEBUG [Thread-1665 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:21,657 DEBUG [Thread-1665 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:21,658 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:21,658 DEBUG [Thread-1650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:21,659 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:21,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:21,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:15:21,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:21,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:15:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:15:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:15:21,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:21,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:21,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:21,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:21,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:21,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:21,672 DEBUG [Thread-1654 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:21,673 DEBUG [Thread-1656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:21,676 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:21,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:15:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:15:21,680 DEBUG [Thread-1658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:21,681 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:15:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-27T21:15:21,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T21:15:21,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:15:21,684 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:15:21,685 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:15:21,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:15:21,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112771ae73bdf5cf4bf29c6f1edd14bdac19_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742121663/Put/seqid=0 2024-11-27T21:15:21,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742190_1366 (size=17034) 2024-11-27T21:15:21,721 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:21,724 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112771ae73bdf5cf4bf29c6f1edd14bdac19_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112771ae73bdf5cf4bf29c6f1edd14bdac19_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:21,725 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/98262bcf170e423c938623c5e1e597e5, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:21,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/98262bcf170e423c938623c5e1e597e5 is 175, key is test_row_0/A:col10/1732742121663/Put/seqid=0 2024-11-27T21:15:21,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742191_1367 (size=48139) 2024-11-27T21:15:21,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:21,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742181713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742181715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:21,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742121707, max timestamp=1732742121732Num uniques 859; 2024-11-27T21:15:21,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:15:21,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742121707, max timestamp=1732742121722Num uniques 859; 2024-11-27T21:15:21,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742121707, max timestamp=1732742121732Num uniques 859; 2024-11-27T21:15:21,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:15:21,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:15:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T21:15:21,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742181801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:21,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742181804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:21,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742181805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:21,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:21,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T21:15:21,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:21,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:21,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:21,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:21,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:21,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742181859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:21,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742181859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:21,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742181919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:21,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:21,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742181929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:21,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742181929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:21,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T21:15:21,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:21,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:21,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:21,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:21,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T21:15:22,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742182079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:22,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742182079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:22,134 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/98262bcf170e423c938623c5e1e597e5 2024-11-27T21:15:22,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T21:15:22,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:22,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:22,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:22,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742182139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:22,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742182140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:22,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742182140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:22,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:22,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/27169f1c17a242febb925744c6080c87 is 50, key is test_row_0/B:col10/1732742121663/Put/seqid=0 2024-11-27T21:15:22,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742192_1368 (size=12001) 2024-11-27T21:15:22,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T21:15:22,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:22,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:22,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:22,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T21:15:22,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742182399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:22,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742182399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:22,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:22,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:22,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T21:15:22,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:22,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:22,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:22,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742182458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:22,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742182458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:22,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742182458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:22,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/27169f1c17a242febb925744c6080c87 2024-11-27T21:15:22,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/e8e02b98566a4947b2608bf12dbcfe2c is 50, key is test_row_0/C:col10/1732742121663/Put/seqid=0 2024-11-27T21:15:22,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742193_1369 (size=12001) 2024-11-27T21:15:22,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T21:15:22,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:22,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:22,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:22,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T21:15:22,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:22,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:22,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:22,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T21:15:22,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T21:15:22,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:22,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:22,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:22,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:22,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742182919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:22,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742182919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:22,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:22,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742182969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:22,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742182978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:22,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742182979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:22,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/e8e02b98566a4947b2608bf12dbcfe2c 2024-11-27T21:15:23,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/98262bcf170e423c938623c5e1e597e5 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/98262bcf170e423c938623c5e1e597e5 2024-11-27T21:15:23,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/98262bcf170e423c938623c5e1e597e5, entries=250, sequenceid=14, filesize=47.0 K 2024-11-27T21:15:23,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/27169f1c17a242febb925744c6080c87 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/27169f1c17a242febb925744c6080c87 2024-11-27T21:15:23,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/27169f1c17a242febb925744c6080c87, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T21:15:23,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/e8e02b98566a4947b2608bf12dbcfe2c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e8e02b98566a4947b2608bf12dbcfe2c 2024-11-27T21:15:23,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e8e02b98566a4947b2608bf12dbcfe2c, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T21:15:23,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 1343ms, sequenceid=14, compaction requested=false 2024-11-27T21:15:23,013 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T21:15:23,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:23,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T21:15:23,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:23,061 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-27T21:15:23,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:23,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bc6e35b4f84c4bd780b77b5530381441_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742121799/Put/seqid=0 2024-11-27T21:15:23,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742194_1370 (size=12154) 2024-11-27T21:15:23,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,077 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bc6e35b4f84c4bd780b77b5530381441_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bc6e35b4f84c4bd780b77b5530381441_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:23,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/3f8c8f98f69744dfa25a7fb3110d8c87, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/3f8c8f98f69744dfa25a7fb3110d8c87 is 175, key is test_row_0/A:col10/1732742121799/Put/seqid=0 2024-11-27T21:15:23,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742195_1371 (size=30955) 2024-11-27T21:15:23,286 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T21:15:23,485 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/3f8c8f98f69744dfa25a7fb3110d8c87 2024-11-27T21:15:23,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/a60b14a921114207be9583eef51647df is 50, key is test_row_0/B:col10/1732742121799/Put/seqid=0 2024-11-27T21:15:23,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742196_1372 (size=12001) 2024-11-27T21:15:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T21:15:23,897 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/a60b14a921114207be9583eef51647df 2024-11-27T21:15:23,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/c25a2f0cd5484f2c9a95a0403d202154 is 50, key is test_row_0/C:col10/1732742121799/Put/seqid=0 2024-11-27T21:15:23,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742197_1373 (size=12001) 2024-11-27T21:15:23,910 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/c25a2f0cd5484f2c9a95a0403d202154 2024-11-27T21:15:23,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/3f8c8f98f69744dfa25a7fb3110d8c87 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/3f8c8f98f69744dfa25a7fb3110d8c87 2024-11-27T21:15:23,917 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/3f8c8f98f69744dfa25a7fb3110d8c87, entries=150, sequenceid=51, filesize=30.2 K 2024-11-27T21:15:23,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/a60b14a921114207be9583eef51647df as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/a60b14a921114207be9583eef51647df 2024-11-27T21:15:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,922 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/a60b14a921114207be9583eef51647df, entries=150, sequenceid=51, filesize=11.7 K 2024-11-27T21:15:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/c25a2f0cd5484f2c9a95a0403d202154 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/c25a2f0cd5484f2c9a95a0403d202154 2024-11-27T21:15:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,929 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/c25a2f0cd5484f2c9a95a0403d202154, entries=150, sequenceid=51, filesize=11.7 K 2024-11-27T21:15:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,930 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 869ms, sequenceid=51, compaction requested=false 2024-11-27T21:15:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:23,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:23,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-27T21:15:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-27T21:15:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-27T21:15:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2470 sec 2024-11-27T21:15:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.2530 sec 2024-11-27T21:15:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:24,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:15:24,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:24,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:24,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:24,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:24,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:24,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127dcd12eb06ce34df7a589f993b4900d4c_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742124046/Put/seqid=0 2024-11-27T21:15:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742198_1374 (size=21918) 2024-11-27T21:15:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742124131, max timestamp=1732742124184Num uniques 859; 2024-11-27T21:15:24,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742124131, max timestamp=1732742124184Num uniques 859; 2024-11-27T21:15:24,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:15:24,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:15:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742184168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742124131, max timestamp=1732742124184Num uniques 859; 2024-11-27T21:15:24,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:15:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742184237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742184240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742184241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742184242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742184299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742184358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742184359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742184369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742184369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,507 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:24,511 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127dcd12eb06ce34df7a589f993b4900d4c_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127dcd12eb06ce34df7a589f993b4900d4c_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:24,512 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/cfd5f8f8a0c84552b26f6683685f673f, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:24,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/cfd5f8f8a0c84552b26f6683685f673f is 175, key is test_row_0/A:col10/1732742124046/Put/seqid=0 2024-11-27T21:15:24,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742199_1375 (size=65323) 2024-11-27T21:15:24,517 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/cfd5f8f8a0c84552b26f6683685f673f 2024-11-27T21:15:24,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742184508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/6f8a39157fc049039a8ad8c8393479f2 is 50, key is test_row_0/B:col10/1732742124046/Put/seqid=0 2024-11-27T21:15:24,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742200_1376 (size=12001) 2024-11-27T21:15:24,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742184569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742184569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742184589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742184589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742184829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742184888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742184889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742184909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742184909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:24,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/6f8a39157fc049039a8ad8c8393479f2 2024-11-27T21:15:24,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/680f7d1b85c54169933d10449bbf80ec is 50, key is test_row_0/C:col10/1732742124046/Put/seqid=0 2024-11-27T21:15:24,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742201_1377 (size=12001) 2024-11-27T21:15:25,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/680f7d1b85c54169933d10449bbf80ec 2024-11-27T21:15:25,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/cfd5f8f8a0c84552b26f6683685f673f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/cfd5f8f8a0c84552b26f6683685f673f 2024-11-27T21:15:25,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/cfd5f8f8a0c84552b26f6683685f673f, entries=350, sequenceid=61, filesize=63.8 K 2024-11-27T21:15:25,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/6f8a39157fc049039a8ad8c8393479f2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/6f8a39157fc049039a8ad8c8393479f2 2024-11-27T21:15:25,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/6f8a39157fc049039a8ad8c8393479f2, entries=150, sequenceid=61, filesize=11.7 K 2024-11-27T21:15:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/680f7d1b85c54169933d10449bbf80ec as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/680f7d1b85c54169933d10449bbf80ec 2024-11-27T21:15:25,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/680f7d1b85c54169933d10449bbf80ec, entries=150, sequenceid=61, filesize=11.7 K 2024-11-27T21:15:25,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 1270ms, sequenceid=61, compaction requested=true 2024-11-27T21:15:25,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:25,360 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:25,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:15:25,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:25,360 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:25,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:15:25,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:25,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:25,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:25,361 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:25,361 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/A is initiating minor compaction (all files) 2024-11-27T21:15:25,361 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/A in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:25,361 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/98262bcf170e423c938623c5e1e597e5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/3f8c8f98f69744dfa25a7fb3110d8c87, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/cfd5f8f8a0c84552b26f6683685f673f] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=141.0 K 2024-11-27T21:15:25,361 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:25,361 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/98262bcf170e423c938623c5e1e597e5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/3f8c8f98f69744dfa25a7fb3110d8c87, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/cfd5f8f8a0c84552b26f6683685f673f] 2024-11-27T21:15:25,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:25,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-27T21:15:25,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:25,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:15:25,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:25,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:15:25,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:25,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:15:25,366 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:25,366 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/B is initiating minor compaction (all files) 2024-11-27T21:15:25,366 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/B in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:25,366 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/27169f1c17a242febb925744c6080c87, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/a60b14a921114207be9583eef51647df, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/6f8a39157fc049039a8ad8c8393479f2] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=35.2 K 2024-11-27T21:15:25,366 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98262bcf170e423c938623c5e1e597e5, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732742121663 2024-11-27T21:15:25,366 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 27169f1c17a242febb925744c6080c87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732742121663 2024-11-27T21:15:25,366 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting a60b14a921114207be9583eef51647df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732742121798 2024-11-27T21:15:25,366 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f8c8f98f69744dfa25a7fb3110d8c87, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732742121798 2024-11-27T21:15:25,367 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfd5f8f8a0c84552b26f6683685f673f, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732742124013 2024-11-27T21:15:25,367 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f8a39157fc049039a8ad8c8393479f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732742124046 2024-11-27T21:15:25,372 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:25,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277fc504e398b84a35b104486954eb0979_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742125360/Put/seqid=0 2024-11-27T21:15:25,377 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411270a995997d4ab4a6789a2cfe9a3cea59c_49ff3edd8f78b405b2b26bdd2aa6e347 store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:25,379 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411270a995997d4ab4a6789a2cfe9a3cea59c_49ff3edd8f78b405b2b26bdd2aa6e347, store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:25,379 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411270a995997d4ab4a6789a2cfe9a3cea59c_49ff3edd8f78b405b2b26bdd2aa6e347 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:25,381 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:25,381 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/15253010c6ac4f0a8639a4165ca79b3f is 50, key is test_row_0/B:col10/1732742124046/Put/seqid=0 2024-11-27T21:15:25,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742202_1378 (size=14594) 2024-11-27T21:15:25,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742203_1379 (size=4469) 2024-11-27T21:15:25,389 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#A#compaction#323 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:25,390 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/4e5e608945c24b729b0423e3be1dedf8 is 175, key is test_row_0/A:col10/1732742124046/Put/seqid=0 2024-11-27T21:15:25,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742205_1381 (size=31058) 2024-11-27T21:15:25,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742204_1380 (size=12104) 2024-11-27T21:15:25,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742185411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:25,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742185422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:25,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742185423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:25,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742185429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:25,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742185429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:25,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742185528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:25,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742185539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:25,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742185549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:25,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742185749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:25,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742185759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:25,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742185769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:25,788 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:25,792 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277fc504e398b84a35b104486954eb0979_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277fc504e398b84a35b104486954eb0979_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:25,793 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/a56d71b2b7e34c3a987efb7dc7cfbad0, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:25,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/a56d71b2b7e34c3a987efb7dc7cfbad0 is 175, key is test_row_0/A:col10/1732742125360/Put/seqid=0 2024-11-27T21:15:25,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742206_1382 (size=39549) 2024-11-27T21:15:25,798 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/a56d71b2b7e34c3a987efb7dc7cfbad0 2024-11-27T21:15:25,798 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/4e5e608945c24b729b0423e3be1dedf8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/4e5e608945c24b729b0423e3be1dedf8 2024-11-27T21:15:25,803 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/A of 49ff3edd8f78b405b2b26bdd2aa6e347 into 4e5e608945c24b729b0423e3be1dedf8(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:25,803 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:25,803 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/A, priority=13, startTime=1732742125359; duration=0sec 2024-11-27T21:15:25,803 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:25,803 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:A 2024-11-27T21:15:25,803 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:25,804 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/15253010c6ac4f0a8639a4165ca79b3f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/15253010c6ac4f0a8639a4165ca79b3f 2024-11-27T21:15:25,805 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:25,805 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/C is initiating minor compaction (all files) 2024-11-27T21:15:25,805 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/C in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:25,805 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e8e02b98566a4947b2608bf12dbcfe2c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/c25a2f0cd5484f2c9a95a0403d202154, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/680f7d1b85c54169933d10449bbf80ec] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=35.2 K 2024-11-27T21:15:25,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/227ecad780054600be37686f717eecbc is 50, key is test_row_0/B:col10/1732742125360/Put/seqid=0 2024-11-27T21:15:25,806 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8e02b98566a4947b2608bf12dbcfe2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732742121663 2024-11-27T21:15:25,807 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting c25a2f0cd5484f2c9a95a0403d202154, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732742121798 2024-11-27T21:15:25,808 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 680f7d1b85c54169933d10449bbf80ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732742124046 2024-11-27T21:15:25,811 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/B of 49ff3edd8f78b405b2b26bdd2aa6e347 into 15253010c6ac4f0a8639a4165ca79b3f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:25,811 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:25,811 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/B, priority=13, startTime=1732742125360; duration=0sec 2024-11-27T21:15:25,811 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:25,811 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:B 2024-11-27T21:15:25,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742207_1383 (size=12001) 2024-11-27T21:15:25,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/227ecad780054600be37686f717eecbc 2024-11-27T21:15:25,818 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#C#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:25,818 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/12f1571f7c754a9c9c801f510857c05c is 50, key is test_row_0/C:col10/1732742124046/Put/seqid=0 2024-11-27T21:15:25,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/33d467ec1e8e4658a938dea45324f3cb is 50, key is test_row_0/C:col10/1732742125360/Put/seqid=0 2024-11-27T21:15:25,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742209_1385 (size=12001) 2024-11-27T21:15:25,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742208_1384 (size=12104) 2024-11-27T21:15:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T21:15:25,849 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:25,850 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:15:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-27T21:15:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T21:15:25,851 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:15:25,852 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:15:25,852 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:15:25,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T21:15:26,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T21:15:26,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:26,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:26,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:26,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:26,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742186059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:26,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742186079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:26,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742186098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:26,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T21:15:26,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:26,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:26,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:26,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:26,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T21:15:26,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/33d467ec1e8e4658a938dea45324f3cb 2024-11-27T21:15:26,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/a56d71b2b7e34c3a987efb7dc7cfbad0 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/a56d71b2b7e34c3a987efb7dc7cfbad0 2024-11-27T21:15:26,239 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/12f1571f7c754a9c9c801f510857c05c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/12f1571f7c754a9c9c801f510857c05c 2024-11-27T21:15:26,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/a56d71b2b7e34c3a987efb7dc7cfbad0, entries=200, sequenceid=99, filesize=38.6 K 2024-11-27T21:15:26,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/227ecad780054600be37686f717eecbc as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/227ecad780054600be37686f717eecbc 2024-11-27T21:15:26,244 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/C of 49ff3edd8f78b405b2b26bdd2aa6e347 into 12f1571f7c754a9c9c801f510857c05c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:26,244 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:26,244 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/C, priority=13, startTime=1732742125360; duration=0sec 2024-11-27T21:15:26,245 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:26,245 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:C 2024-11-27T21:15:26,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/227ecad780054600be37686f717eecbc, entries=150, sequenceid=99, filesize=11.7 K 2024-11-27T21:15:26,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/33d467ec1e8e4658a938dea45324f3cb as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/33d467ec1e8e4658a938dea45324f3cb 2024-11-27T21:15:26,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/33d467ec1e8e4658a938dea45324f3cb, entries=150, sequenceid=99, filesize=11.7 K 2024-11-27T21:15:26,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 890ms, sequenceid=99, compaction requested=false 2024-11-27T21:15:26,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:26,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T21:15:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:26,311 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-27T21:15:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:26,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127cf5395e67388401e91520cd3a9450c45_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742125421/Put/seqid=0 2024-11-27T21:15:26,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742210_1386 (size=12154) 2024-11-27T21:15:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:26,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T21:15:26,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742126511, max timestamp=1732742126551Num uniques 859; 2024-11-27T21:15:26,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:15:26,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742126511, max timestamp=1732742126551Num uniques 859; 2024-11-27T21:15:26,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:15:26,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742126511, max timestamp=1732742126551Num uniques 859; 2024-11-27T21:15:26,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:15:26,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742186618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:26,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742186618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:26,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742186619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:26,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742186620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:26,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742186630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:26,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:26,734 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127cf5395e67388401e91520cd3a9450c45_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cf5395e67388401e91520cd3a9450c45_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:26,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/29176966dcea429f84882f5cf3d82633, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:26,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/29176966dcea429f84882f5cf3d82633 is 175, key is test_row_0/A:col10/1732742125421/Put/seqid=0 2024-11-27T21:15:26,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742186738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:26,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742186739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742186739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:26,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742211_1387 (size=30955) 2024-11-27T21:15:26,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742186740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:26,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742186949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:26,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742186949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:26,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742186949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:26,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742186959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T21:15:27,150 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/29176966dcea429f84882f5cf3d82633 2024-11-27T21:15:27,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/eed92e63c69c48d680d50ab6905e5dad is 50, key is test_row_0/B:col10/1732742125421/Put/seqid=0 2024-11-27T21:15:27,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742212_1388 (size=12001) 2024-11-27T21:15:27,161 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/eed92e63c69c48d680d50ab6905e5dad 2024-11-27T21:15:27,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/ae73a1d7178a4827b9a71216e4ded7db is 50, key is test_row_0/C:col10/1732742125421/Put/seqid=0 2024-11-27T21:15:27,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742213_1389 (size=12001) 2024-11-27T21:15:27,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742187268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:27,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:27,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742187269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:27,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:27,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:27,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742187269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:27,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:27,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742187279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:27,572 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/ae73a1d7178a4827b9a71216e4ded7db 2024-11-27T21:15:27,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/29176966dcea429f84882f5cf3d82633 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/29176966dcea429f84882f5cf3d82633 2024-11-27T21:15:27,581 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/29176966dcea429f84882f5cf3d82633, entries=150, sequenceid=111, filesize=30.2 K 2024-11-27T21:15:27,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/eed92e63c69c48d680d50ab6905e5dad as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/eed92e63c69c48d680d50ab6905e5dad 2024-11-27T21:15:27,585 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/eed92e63c69c48d680d50ab6905e5dad, entries=150, sequenceid=111, filesize=11.7 K 2024-11-27T21:15:27,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/ae73a1d7178a4827b9a71216e4ded7db as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/ae73a1d7178a4827b9a71216e4ded7db 2024-11-27T21:15:27,589 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/ae73a1d7178a4827b9a71216e4ded7db, entries=150, sequenceid=111, filesize=11.7 K 2024-11-27T21:15:27,590 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 1279ms, sequenceid=111, compaction requested=true 2024-11-27T21:15:27,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:27,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:27,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-27T21:15:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-27T21:15:27,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-27T21:15:27,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7400 sec 2024-11-27T21:15:27,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.7450 sec 2024-11-27T21:15:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:27,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-27T21:15:27,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:27,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:15:27,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:27,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:15:27,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:27,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:15:27,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127876cca9c945f4d9db8514c7e7581642e_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742126629/Put/seqid=0 2024-11-27T21:15:27,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742214_1390 (size=14794) 2024-11-27T21:15:27,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742187692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:27,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742187789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:27,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742187789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:27,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742187789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:27,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742187799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:27,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742187808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T21:15:27,999 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:27,999 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:15:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-27T21:15:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T21:15:28,001 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:15:28,002 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:15:28,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:15:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742188019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:28,057 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,061 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127876cca9c945f4d9db8514c7e7581642e_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127876cca9c945f4d9db8514c7e7581642e_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:28,062 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/3c01f32c4147429aa82549dcaa894a75, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:28,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/3c01f32c4147429aa82549dcaa894a75 is 175, key is test_row_0/A:col10/1732742126629/Put/seqid=0 2024-11-27T21:15:28,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742215_1391 (size=39749) 2024-11-27T21:15:28,066 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/3c01f32c4147429aa82549dcaa894a75 2024-11-27T21:15:28,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/63ff1960edd44d9c804a178318b04b1f is 50, key is test_row_0/B:col10/1732742126629/Put/seqid=0 2024-11-27T21:15:28,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742216_1392 (size=12151) 2024-11-27T21:15:28,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/63ff1960edd44d9c804a178318b04b1f 2024-11-27T21:15:28,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/71ca25a6d1ac4a7f973bc9bdbeb070a3 is 50, key is test_row_0/C:col10/1732742126629/Put/seqid=0 2024-11-27T21:15:28,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742217_1393 (size=12151) 2024-11-27T21:15:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T21:15:28,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T21:15:28,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:28,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:28,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:28,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:28,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:28,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T21:15:28,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:28,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:28,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:28,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:28,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T21:15:28,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742188339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:28,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T21:15:28,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:28,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:28,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:28,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:28,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:28,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/71ca25a6d1ac4a7f973bc9bdbeb070a3 2024-11-27T21:15:28,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/3c01f32c4147429aa82549dcaa894a75 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/3c01f32c4147429aa82549dcaa894a75 2024-11-27T21:15:28,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/3c01f32c4147429aa82549dcaa894a75, entries=200, sequenceid=149, filesize=38.8 K 2024-11-27T21:15:28,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/63ff1960edd44d9c804a178318b04b1f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/63ff1960edd44d9c804a178318b04b1f 2024-11-27T21:15:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/63ff1960edd44d9c804a178318b04b1f, entries=150, sequenceid=149, filesize=11.9 K 2024-11-27T21:15:28,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/71ca25a6d1ac4a7f973bc9bdbeb070a3 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/71ca25a6d1ac4a7f973bc9bdbeb070a3 2024-11-27T21:15:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/71ca25a6d1ac4a7f973bc9bdbeb070a3, entries=150, sequenceid=149, filesize=11.9 K 2024-11-27T21:15:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 866ms, sequenceid=149, compaction requested=true 2024-11-27T21:15:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,511 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:15:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:15:28,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,512 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:15:28,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:15:28,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:28,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:28,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,513 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141311 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:15:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,513 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/A is initiating minor compaction (all files) 2024-11-27T21:15:28,513 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/A in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,513 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/4e5e608945c24b729b0423e3be1dedf8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/a56d71b2b7e34c3a987efb7dc7cfbad0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/29176966dcea429f84882f5cf3d82633, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/3c01f32c4147429aa82549dcaa894a75] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=138.0 K 2024-11-27T21:15:28,513 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:28,513 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/4e5e608945c24b729b0423e3be1dedf8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/a56d71b2b7e34c3a987efb7dc7cfbad0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/29176966dcea429f84882f5cf3d82633, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/3c01f32c4147429aa82549dcaa894a75] 2024-11-27T21:15:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,513 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:15:28,513 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e5e608945c24b729b0423e3be1dedf8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732742124046 2024-11-27T21:15:28,513 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/B is initiating minor compaction (all files) 2024-11-27T21:15:28,513 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/B in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:28,514 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/15253010c6ac4f0a8639a4165ca79b3f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/227ecad780054600be37686f717eecbc, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/eed92e63c69c48d680d50ab6905e5dad, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/63ff1960edd44d9c804a178318b04b1f] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=47.1 K 2024-11-27T21:15:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,514 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting a56d71b2b7e34c3a987efb7dc7cfbad0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732742124218 2024-11-27T21:15:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,514 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 15253010c6ac4f0a8639a4165ca79b3f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732742124046 2024-11-27T21:15:28,514 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29176966dcea429f84882f5cf3d82633, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732742125401 2024-11-27T21:15:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,514 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 227ecad780054600be37686f717eecbc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732742124236 2024-11-27T21:15:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,514 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c01f32c4147429aa82549dcaa894a75, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732742126551 2024-11-27T21:15:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,515 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting eed92e63c69c48d680d50ab6905e5dad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732742125401 2024-11-27T21:15:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,515 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 63ff1960edd44d9c804a178318b04b1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732742126619 2024-11-27T21:15:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,522 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,525 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#B#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,525 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/34493e801642465ba24aef70f5ca1f6e is 50, key is test_row_0/B:col10/1732742126629/Put/seqid=0 2024-11-27T21:15:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,527 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127258c266f3f474e2a9013a115981faed2_49ff3edd8f78b405b2b26bdd2aa6e347 store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,530 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127258c266f3f474e2a9013a115981faed2_49ff3edd8f78b405b2b26bdd2aa6e347, store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:28,530 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127258c266f3f474e2a9013a115981faed2_49ff3edd8f78b405b2b26bdd2aa6e347 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742218_1394 (size=12391) 2024-11-27T21:15:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742219_1395 (size=4469) 2024-11-27T21:15:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T21:15:28,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,614 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-27T21:15:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:28,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:28,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:28,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:28,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:28,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T21:15:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411270f598fea251e4d2ea0ee7ae33c7048cc_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742127690/Put/seqid=0 2024-11-27T21:15:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742220_1396 (size=9814) 2024-11-27T21:15:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:28,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732742128857, max timestamp=1732742128893Num uniques 859; 2024-11-27T21:15:28,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732742128857, max timestamp=1732742128893Num uniques 859; 2024-11-27T21:15:28,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-27T21:15:28,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-27T21:15:28,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732742128857, max timestamp=1732742128893Num uniques 859; 2024-11-27T21:15:28,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-27T21:15:28,938 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/34493e801642465ba24aef70f5ca1f6e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/34493e801642465ba24aef70f5ca1f6e 2024-11-27T21:15:28,942 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/B of 49ff3edd8f78b405b2b26bdd2aa6e347 into 34493e801642465ba24aef70f5ca1f6e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:28,942 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:28,942 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/B, priority=12, startTime=1732742128512; duration=0sec 2024-11-27T21:15:28,942 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:28,942 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:B 2024-11-27T21:15:28,942 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:15:28,943 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:15:28,943 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/C is initiating minor compaction (all files) 2024-11-27T21:15:28,943 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/C in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:28,943 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/12f1571f7c754a9c9c801f510857c05c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/33d467ec1e8e4658a938dea45324f3cb, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/ae73a1d7178a4827b9a71216e4ded7db, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/71ca25a6d1ac4a7f973bc9bdbeb070a3] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=47.1 K 2024-11-27T21:15:28,944 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 12f1571f7c754a9c9c801f510857c05c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732742124046 2024-11-27T21:15:28,944 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 33d467ec1e8e4658a938dea45324f3cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732742124236 2024-11-27T21:15:28,944 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting ae73a1d7178a4827b9a71216e4ded7db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732742125401 2024-11-27T21:15:28,945 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 71ca25a6d1ac4a7f973bc9bdbeb070a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732742126619 2024-11-27T21:15:28,948 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#A#compaction#334 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:28,948 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/f42fa8b58a9c4d40abe89bb2c1eb7c2e is 175, key is test_row_0/A:col10/1732742126629/Put/seqid=0 2024-11-27T21:15:28,952 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#C#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:28,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742221_1397 (size=31345) 2024-11-27T21:15:28,952 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/157e7cedf5dc41f098633b0cd753062e is 50, key is test_row_0/C:col10/1732742126629/Put/seqid=0 2024-11-27T21:15:28,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742222_1398 (size=12391) 2024-11-27T21:15:28,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742188941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:28,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742188975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:28,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742188975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:28,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742188975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:28,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742188976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:29,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:29,050 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411270f598fea251e4d2ea0ee7ae33c7048cc_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270f598fea251e4d2ea0ee7ae33c7048cc_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:29,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/9fef0041ecac48b2b5a8581922fbcb70, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:29,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/9fef0041ecac48b2b5a8581922fbcb70 is 175, key is test_row_0/A:col10/1732742127690/Put/seqid=0 2024-11-27T21:15:29,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742223_1399 (size=22461) 2024-11-27T21:15:29,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742189079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:29,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:29,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742189099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:29,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:29,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742189099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:29,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742189099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:29,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742189099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T21:15:29,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742189299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:29,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742189308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:29,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742189308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:29,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742189309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:29,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742189309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:29,356 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/f42fa8b58a9c4d40abe89bb2c1eb7c2e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/f42fa8b58a9c4d40abe89bb2c1eb7c2e 2024-11-27T21:15:29,360 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/157e7cedf5dc41f098633b0cd753062e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/157e7cedf5dc41f098633b0cd753062e 2024-11-27T21:15:29,361 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/A of 49ff3edd8f78b405b2b26bdd2aa6e347 into f42fa8b58a9c4d40abe89bb2c1eb7c2e(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:29,361 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:29,361 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/A, priority=12, startTime=1732742128511; duration=0sec 2024-11-27T21:15:29,361 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:29,361 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:A 2024-11-27T21:15:29,364 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/C of 49ff3edd8f78b405b2b26bdd2aa6e347 into 157e7cedf5dc41f098633b0cd753062e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:29,364 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:29,364 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/C, priority=12, startTime=1732742128512; duration=0sec 2024-11-27T21:15:29,365 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:29,365 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:C 2024-11-27T21:15:29,455 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/9fef0041ecac48b2b5a8581922fbcb70 2024-11-27T21:15:29,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/feb968b357af457090edf36bc388cfe2 is 50, key is test_row_0/B:col10/1732742127690/Put/seqid=0 2024-11-27T21:15:29,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742224_1400 (size=9757) 2024-11-27T21:15:29,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742189609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:29,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742189629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:29,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742189629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:29,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742189629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:29,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742189629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:29,865 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/feb968b357af457090edf36bc388cfe2 2024-11-27T21:15:29,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/9fa1133c50024445b3d26c61bead74c1 is 50, key is test_row_0/C:col10/1732742127690/Put/seqid=0 2024-11-27T21:15:29,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742225_1401 (size=9757) 2024-11-27T21:15:30,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742190129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T21:15:30,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742190149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:30,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742190149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:30,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742190149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:30,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742190149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:30,277 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/9fa1133c50024445b3d26c61bead74c1 2024-11-27T21:15:30,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/9fef0041ecac48b2b5a8581922fbcb70 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9fef0041ecac48b2b5a8581922fbcb70 2024-11-27T21:15:30,285 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9fef0041ecac48b2b5a8581922fbcb70, entries=100, sequenceid=157, filesize=21.9 K 2024-11-27T21:15:30,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/feb968b357af457090edf36bc388cfe2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/feb968b357af457090edf36bc388cfe2 2024-11-27T21:15:30,289 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/feb968b357af457090edf36bc388cfe2, entries=100, sequenceid=157, filesize=9.5 K 2024-11-27T21:15:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/9fa1133c50024445b3d26c61bead74c1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/9fa1133c50024445b3d26c61bead74c1 2024-11-27T21:15:30,293 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/9fa1133c50024445b3d26c61bead74c1, entries=100, sequenceid=157, filesize=9.5 K 2024-11-27T21:15:30,293 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 1679ms, sequenceid=157, compaction requested=false 2024-11-27T21:15:30,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:30,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:30,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-27T21:15:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-27T21:15:30,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-27T21:15:30,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2930 sec 2024-11-27T21:15:30,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.2960 sec 2024-11-27T21:15:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:31,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=161.02 KB heapSize=451.31 KB 2024-11-27T21:15:31,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:31,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:15:31,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:31,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:15:31,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:31,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:15:31,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274d4237aa03634a75bf94f888b2547fe7_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742128952/Put/seqid=0 2024-11-27T21:15:31,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742226_1402 (size=14794) 2024-11-27T21:15:31,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742191169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:31,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742191170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:31,182 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_0' in 49ff3edd8f78b405b2b26bdd2aa6e347 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:15:31,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,183 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_0' in 49ff3edd8f78b405b2b26bdd2aa6e347 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:15:31,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:31,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:31,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742191180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:31,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742191181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742191182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:31,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742191299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:31,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:31,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742191299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:31,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:31,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742191299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:31,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742191509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:31,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:31,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742191509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:31,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742191509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:31,578 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:31,595 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274d4237aa03634a75bf94f888b2547fe7_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274d4237aa03634a75bf94f888b2547fe7_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:31,597 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/8df47496313a4ee3be45b6db260e07cf, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:31,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/8df47496313a4ee3be45b6db260e07cf is 175, key is test_row_0/A:col10/1732742128952/Put/seqid=0 2024-11-27T21:15:31,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742227_1403 (size=39749) 2024-11-27T21:15:31,631 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/8df47496313a4ee3be45b6db260e07cf 2024-11-27T21:15:31,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/15fe4cc06a0a4a7784e1f2db3d0f1bc1 is 50, key is test_row_0/B:col10/1732742128952/Put/seqid=0 2024-11-27T21:15:31,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742228_1404 (size=12151) 2024-11-27T21:15:31,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/15fe4cc06a0a4a7784e1f2db3d0f1bc1 2024-11-27T21:15:31,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/e4cf5b05a22d40bd92377d9a337ad8f2 is 50, key is test_row_0/C:col10/1732742128952/Put/seqid=0 2024-11-27T21:15:31,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742229_1405 (size=12151) 2024-11-27T21:15:31,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742191829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:31,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742191829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:31,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:31,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742191830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:31,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:31,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:32,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/e4cf5b05a22d40bd92377d9a337ad8f2 2024-11-27T21:15:32,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/8df47496313a4ee3be45b6db260e07cf as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/8df47496313a4ee3be45b6db260e07cf 2024-11-27T21:15:32,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/8df47496313a4ee3be45b6db260e07cf, entries=200, sequenceid=198, filesize=38.8 K 2024-11-27T21:15:32,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/15fe4cc06a0a4a7784e1f2db3d0f1bc1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/15fe4cc06a0a4a7784e1f2db3d0f1bc1 2024-11-27T21:15:32,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/15fe4cc06a0a4a7784e1f2db3d0f1bc1, entries=150, sequenceid=198, filesize=11.9 K 2024-11-27T21:15:32,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/e4cf5b05a22d40bd92377d9a337ad8f2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e4cf5b05a22d40bd92377d9a337ad8f2 2024-11-27T21:15:32,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e4cf5b05a22d40bd92377d9a337ad8f2, entries=150, sequenceid=198, filesize=11.9 K 2024-11-27T21:15:32,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~451.27 KB/462096, currentSize=26.84 KB/27480 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 951ms, sequenceid=198, compaction requested=true 2024-11-27T21:15:32,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:32,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:15:32,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:32,109 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:32,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:15:32,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:32,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:32,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:32,109 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:32,110 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:32,110 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/A is initiating minor compaction (all files) 2024-11-27T21:15:32,110 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/A in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:32,110 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/f42fa8b58a9c4d40abe89bb2c1eb7c2e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9fef0041ecac48b2b5a8581922fbcb70, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/8df47496313a4ee3be45b6db260e07cf] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=91.4 K 2024-11-27T21:15:32,110 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:32,111 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/f42fa8b58a9c4d40abe89bb2c1eb7c2e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9fef0041ecac48b2b5a8581922fbcb70, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/8df47496313a4ee3be45b6db260e07cf] 2024-11-27T21:15:32,111 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:32,111 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/B is initiating minor compaction (all files) 2024-11-27T21:15:32,111 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting f42fa8b58a9c4d40abe89bb2c1eb7c2e, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732742126619 2024-11-27T21:15:32,111 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/B in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:32,111 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/34493e801642465ba24aef70f5ca1f6e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/feb968b357af457090edf36bc388cfe2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/15fe4cc06a0a4a7784e1f2db3d0f1bc1] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=33.5 K 2024-11-27T21:15:32,111 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fef0041ecac48b2b5a8581922fbcb70, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732742127681 2024-11-27T21:15:32,111 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 34493e801642465ba24aef70f5ca1f6e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732742126619 2024-11-27T21:15:32,112 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8df47496313a4ee3be45b6db260e07cf, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732742128922 2024-11-27T21:15:32,112 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting feb968b357af457090edf36bc388cfe2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732742127681 2024-11-27T21:15:32,112 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 15fe4cc06a0a4a7784e1f2db3d0f1bc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732742128922 2024-11-27T21:15:32,120 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#B#compaction#343 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:32,120 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/bc027f815cce4bee86b15c87607b6e03 is 50, key is test_row_0/B:col10/1732742128952/Put/seqid=0 2024-11-27T21:15:32,121 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:32,124 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112721a057a6bd914fa582b3702a5df7d7c8_49ff3edd8f78b405b2b26bdd2aa6e347 store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:32,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742230_1406 (size=12493) 2024-11-27T21:15:32,126 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112721a057a6bd914fa582b3702a5df7d7c8_49ff3edd8f78b405b2b26bdd2aa6e347, store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:32,126 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112721a057a6bd914fa582b3702a5df7d7c8_49ff3edd8f78b405b2b26bdd2aa6e347 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:32,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742231_1407 (size=4469) 2024-11-27T21:15:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T21:15:32,159 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:32,159 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:15:32,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-27T21:15:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T21:15:32,161 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:15:32,162 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:15:32,162 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:15:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T21:15:32,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T21:15:32,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:32,315 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-27T21:15:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:32,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127eaf04c345f3e46f0a79164e1a4314e08_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742131181/Put/seqid=0 2024-11-27T21:15:32,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742232_1408 (size=12304) 2024-11-27T21:15:32,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:32,346 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127eaf04c345f3e46f0a79164e1a4314e08_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127eaf04c345f3e46f0a79164e1a4314e08_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:32,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/6a80bb0e228344cf85afd62f6c2a9d0b, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:32,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/6a80bb0e228344cf85afd62f6c2a9d0b is 175, key is test_row_0/A:col10/1732742131181/Put/seqid=0 2024-11-27T21:15:32,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:32,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:32,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742233_1409 (size=31105) 2024-11-27T21:15:32,359 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/6a80bb0e228344cf85afd62f6c2a9d0b 2024-11-27T21:15:32,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/bced16ecab504b6fb62a3d4ccfff4b75 is 50, key is test_row_0/B:col10/1732742131181/Put/seqid=0 2024-11-27T21:15:32,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742234_1410 (size=12151) 2024-11-27T21:15:32,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742132379, max timestamp=1732742132414Num uniques 859; 2024-11-27T21:15:32,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742132379, max timestamp=1732742132414Num uniques 859; 2024-11-27T21:15:32,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:15:32,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:15:32,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742132379, max timestamp=1732742132414Num uniques 859; 2024-11-27T21:15:32,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:15:32,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T21:15:32,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732742132462, max timestamp=1732742132488Num uniques 1118; 2024-11-27T21:15:32,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-27T21:15:32,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732742132462, max timestamp=1732742132488Num uniques 1118; 2024-11-27T21:15:32,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-27T21:15:32,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732742132462, max timestamp=1732742132488Num uniques 1118; 2024-11-27T21:15:32,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-27T21:15:32,530 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/bc027f815cce4bee86b15c87607b6e03 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/bc027f815cce4bee86b15c87607b6e03 2024-11-27T21:15:32,537 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/B of 49ff3edd8f78b405b2b26bdd2aa6e347 into bc027f815cce4bee86b15c87607b6e03(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:32,537 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:32,537 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/B, priority=13, startTime=1732742132109; duration=0sec 2024-11-27T21:15:32,537 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:32,537 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:B 2024-11-27T21:15:32,538 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:32,538 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:32,538 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/C is initiating minor compaction (all files) 2024-11-27T21:15:32,538 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/C in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:32,538 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/157e7cedf5dc41f098633b0cd753062e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/9fa1133c50024445b3d26c61bead74c1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e4cf5b05a22d40bd92377d9a337ad8f2] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=33.5 K 2024-11-27T21:15:32,539 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 157e7cedf5dc41f098633b0cd753062e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732742126619 2024-11-27T21:15:32,539 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fa1133c50024445b3d26c61bead74c1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732742127681 2024-11-27T21:15:32,539 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting e4cf5b05a22d40bd92377d9a337ad8f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732742128922 2024-11-27T21:15:32,545 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#A#compaction#344 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:32,546 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#C#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:32,546 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/e5331d2b245f4394980cf3f165d24847 is 175, key is test_row_0/A:col10/1732742128952/Put/seqid=0 2024-11-27T21:15:32,546 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/699f213ab65c416ab00444e508692c1a is 50, key is test_row_0/C:col10/1732742128952/Put/seqid=0 2024-11-27T21:15:32,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742235_1411 (size=31447) 2024-11-27T21:15:32,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742236_1412 (size=12493) 2024-11-27T21:15:32,556 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/699f213ab65c416ab00444e508692c1a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/699f213ab65c416ab00444e508692c1a 2024-11-27T21:15:32,560 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/C of 49ff3edd8f78b405b2b26bdd2aa6e347 into 699f213ab65c416ab00444e508692c1a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:32,560 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:32,560 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/C, priority=13, startTime=1732742132109; duration=0sec 2024-11-27T21:15:32,560 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:32,560 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:C 2024-11-27T21:15:32,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742192558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:32,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732742132520, max timestamp=1732742132570Num uniques 1127; 2024-11-27T21:15:32,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-27T21:15:32,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732742132520, max timestamp=1732742132570Num uniques 1127; 2024-11-27T21:15:32,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-27T21:15:32,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732742132520, max timestamp=1732742132570Num uniques 1127; 2024-11-27T21:15:32,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-27T21:15:32,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742192678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:32,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732742132657, max timestamp=1732742132691Num uniques 1086; 2024-11-27T21:15:32,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-27T21:15:32,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732742132657, max timestamp=1732742132691Num uniques 1086; 2024-11-27T21:15:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:32,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-27T21:15:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:32,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732742132657, max timestamp=1732742132691Num uniques 1086; 2024-11-27T21:15:32,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-27T21:15:32,774 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/bced16ecab504b6fb62a3d4ccfff4b75 2024-11-27T21:15:32,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/8596a71aa7bd4756a358cf48d1e761e2 is 50, key is test_row_0/C:col10/1732742131181/Put/seqid=0 2024-11-27T21:15:32,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742237_1413 (size=12151) 2024-11-27T21:15:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T21:15:32,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732742132744, max timestamp=1732742132788Num uniques 1045; 2024-11-27T21:15:32,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732742132744, max timestamp=1732742132788Num uniques 1045; 2024-11-27T21:15:32,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-27T21:15:32,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-27T21:15:32,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732742132744, max timestamp=1732742132788Num uniques 1045; 2024-11-27T21:15:32,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-27T21:15:32,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742192900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:32,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732742132866, max timestamp=1732742132900Num uniques 1104; 2024-11-27T21:15:32,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-27T21:15:32,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732742132866, max timestamp=1732742132900Num uniques 1104; 2024-11-27T21:15:32,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-27T21:15:32,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732742132866, max timestamp=1732742132900Num uniques 1104; 2024-11-27T21:15:32,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-27T21:15:32,954 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/e5331d2b245f4394980cf3f165d24847 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/e5331d2b245f4394980cf3f165d24847 2024-11-27T21:15:32,958 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/A of 49ff3edd8f78b405b2b26bdd2aa6e347 into e5331d2b245f4394980cf3f165d24847(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:32,958 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:32,958 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/A, priority=13, startTime=1732742132109; duration=0sec 2024-11-27T21:15:32,958 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:32,958 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:A 2024-11-27T21:15:33,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742193013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732742132964, max timestamp=1732742133013Num uniques 1063; 2024-11-27T21:15:33,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-27T21:15:33,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732742132964, max timestamp=1732742133013Num uniques 1063; 2024-11-27T21:15:33,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732742132964, max timestamp=1732742133013Num uniques 1063; 2024-11-27T21:15:33,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-27T21:15:33,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-27T21:15:33,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732742133064, max timestamp=1732742133109Num uniques 1072; 2024-11-27T21:15:33,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732742133064, max timestamp=1732742133109Num uniques 1072; 2024-11-27T21:15:33,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-27T21:15:33,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-27T21:15:33,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732742133064, max timestamp=1732742133109Num uniques 1072; 2024-11-27T21:15:33,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-27T21:15:33,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1732742133160, max timestamp=1732742133178Num uniques 1031; 2024-11-27T21:15:33,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-11-27T21:15:33,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1732742133160, max timestamp=1732742133178Num uniques 1031; 2024-11-27T21:15:33,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1732742133160, max timestamp=1732742133178Num uniques 1031; 2024-11-27T21:15:33,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-11-27T21:15:33,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-11-27T21:15:33,184 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/8596a71aa7bd4756a358cf48d1e761e2 2024-11-27T21:15:33,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/6a80bb0e228344cf85afd62f6c2a9d0b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6a80bb0e228344cf85afd62f6c2a9d0b 2024-11-27T21:15:33,192 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6a80bb0e228344cf85afd62f6c2a9d0b, entries=150, sequenceid=205, filesize=30.4 K 2024-11-27T21:15:33,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/bced16ecab504b6fb62a3d4ccfff4b75 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/bced16ecab504b6fb62a3d4ccfff4b75 2024-11-27T21:15:33,197 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/bced16ecab504b6fb62a3d4ccfff4b75, entries=150, sequenceid=205, filesize=11.9 K 2024-11-27T21:15:33,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/8596a71aa7bd4756a358cf48d1e761e2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/8596a71aa7bd4756a358cf48d1e761e2 2024-11-27T21:15:33,201 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/8596a71aa7bd4756a358cf48d1e761e2, entries=150, sequenceid=205, filesize=11.9 K 2024-11-27T21:15:33,202 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=60.38 KB/61830 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 887ms, sequenceid=205, compaction requested=false 2024-11-27T21:15:33,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:33,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:33,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:33,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-27T21:15:33,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=67.09 KB heapSize=174.33 KB 2024-11-27T21:15:33,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:33,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:33,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:33,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:33,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:33,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-27T21:15:33,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:33,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-27T21:15:33,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0420 sec 2024-11-27T21:15:33,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.0480 sec 2024-11-27T21:15:33,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127564da156e8684393b1f39c562e56d61b_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742133201/Put/seqid=0 2024-11-27T21:15:33,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742238_1414 (size=14994) 2024-11-27T21:15:33,215 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:33,218 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127564da156e8684393b1f39c562e56d61b_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127564da156e8684393b1f39c562e56d61b_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:33,219 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/171064c00e70477bbb98a2df634cdc28, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:33,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/171064c00e70477bbb98a2df634cdc28 is 175, key is test_row_0/A:col10/1732742133201/Put/seqid=0 2024-11-27T21:15:33,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742239_1415 (size=39949) 2024-11-27T21:15:33,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742193243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742193244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742193251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742193251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742193252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T21:15:33,299 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:33,300 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:15:33,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-27T21:15:33,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T21:15:33,301 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:15:33,302 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:15:33,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:15:33,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742193359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742193369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742193369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742193369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742193379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T21:15:33,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T21:15:33,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:33,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:33,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:33,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:33,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:33,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:33,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742193579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742193579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742193579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742193588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742193599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T21:15:33,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:33,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:33,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:33,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:33,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T21:15:33,623 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/171064c00e70477bbb98a2df634cdc28 2024-11-27T21:15:33,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/7bad767d277049edb219e6afdd68424d is 50, key is test_row_0/B:col10/1732742133201/Put/seqid=0 2024-11-27T21:15:33,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742240_1416 (size=12301) 2024-11-27T21:15:33,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T21:15:33,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:33,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:33,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:33,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:33,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:33,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742193899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742193899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742193899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T21:15:33,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:33,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:33,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:33,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:33,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:33,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742193909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742193909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T21:15:34,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/7bad767d277049edb219e6afdd68424d 2024-11-27T21:15:34,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/e2f371677a8f4cfd87291d3a5edb18c6 is 50, key is test_row_0/C:col10/1732742133201/Put/seqid=0 2024-11-27T21:15:34,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742241_1417 (size=12301) 2024-11-27T21:15:34,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T21:15:34,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:34,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:34,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:34,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:34,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:34,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:34,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T21:15:34,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:34,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:34,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:34,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:34,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:34,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:34,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T21:15:34,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:34,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:34,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:34,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:34,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:34,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:34,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:34,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742194419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:34,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:34,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742194419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:34,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:34,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:34,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:34,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:34,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:34,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742194419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:34,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:34,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:34,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:34,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:34,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:34,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T21:15:34,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:34,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742194429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:34,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:34,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:34,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742194429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:34,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:34,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:34,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:34,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:34,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/e2f371677a8f4cfd87291d3a5edb18c6 2024-11-27T21:15:34,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/171064c00e70477bbb98a2df634cdc28 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/171064c00e70477bbb98a2df634cdc28 2024-11-27T21:15:34,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/171064c00e70477bbb98a2df634cdc28, entries=200, sequenceid=370, filesize=39.0 K 2024-11-27T21:15:34,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/7bad767d277049edb219e6afdd68424d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/7bad767d277049edb219e6afdd68424d 2024-11-27T21:15:34,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/7bad767d277049edb219e6afdd68424d, entries=150, sequenceid=370, filesize=12.0 K 2024-11-27T21:15:34,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/e2f371677a8f4cfd87291d3a5edb18c6 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e2f371677a8f4cfd87291d3a5edb18c6 2024-11-27T21:15:34,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e2f371677a8f4cfd87291d3a5edb18c6, entries=150, sequenceid=370, filesize=12.0 K 2024-11-27T21:15:34,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~174.28 KB/178464, currentSize=114.05 KB/116790 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 1261ms, sequenceid=370, compaction requested=true 2024-11-27T21:15:34,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:34,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:15:34,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:34,464 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:34,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:15:34,464 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:34,465 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:34,465 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/A is initiating minor compaction (all files) 2024-11-27T21:15:34,465 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/A in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:34,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:34,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:34,465 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/e5331d2b245f4394980cf3f165d24847, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6a80bb0e228344cf85afd62f6c2a9d0b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/171064c00e70477bbb98a2df634cdc28] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=100.1 K 2024-11-27T21:15:34,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:34,465 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:34,465 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/e5331d2b245f4394980cf3f165d24847, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6a80bb0e228344cf85afd62f6c2a9d0b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/171064c00e70477bbb98a2df634cdc28] 2024-11-27T21:15:34,465 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:34,465 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/B is initiating minor compaction (all files) 2024-11-27T21:15:34,465 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/B in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:34,465 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/bc027f815cce4bee86b15c87607b6e03, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/bced16ecab504b6fb62a3d4ccfff4b75, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/7bad767d277049edb219e6afdd68424d] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=36.1 K 2024-11-27T21:15:34,466 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5331d2b245f4394980cf3f165d24847, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732742128922 2024-11-27T21:15:34,466 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting bc027f815cce4bee86b15c87607b6e03, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732742128922 2024-11-27T21:15:34,466 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a80bb0e228344cf85afd62f6c2a9d0b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732742131178 2024-11-27T21:15:34,466 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting bced16ecab504b6fb62a3d4ccfff4b75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732742131178 2024-11-27T21:15:34,466 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 171064c00e70477bbb98a2df634cdc28, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732742133176 2024-11-27T21:15:34,466 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bad767d277049edb219e6afdd68424d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732742133176 2024-11-27T21:15:34,475 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:34,477 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127c15adb0b81204e8cb315162a7669c75b_49ff3edd8f78b405b2b26bdd2aa6e347 store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:34,478 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#B#compaction#353 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:34,478 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127c15adb0b81204e8cb315162a7669c75b_49ff3edd8f78b405b2b26bdd2aa6e347, store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:34,478 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c15adb0b81204e8cb315162a7669c75b_49ff3edd8f78b405b2b26bdd2aa6e347 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:34,478 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/c8e544296fc3436195cd0749990d0cf8 is 50, key is test_row_0/B:col10/1732742133201/Put/seqid=0 2024-11-27T21:15:34,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742242_1418 (size=12745) 2024-11-27T21:15:34,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742243_1419 (size=4469) 2024-11-27T21:15:34,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T21:15:34,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:34,526 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-27T21:15:34,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:34,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:34,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:34,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:34,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:34,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:34,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277bc3ed3a1bb24e12aacfb99cae568614_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742133250/Put/seqid=0 2024-11-27T21:15:34,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742244_1420 (size=12454) 2024-11-27T21:15:34,891 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/c8e544296fc3436195cd0749990d0cf8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/c8e544296fc3436195cd0749990d0cf8 2024-11-27T21:15:34,897 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/B of 49ff3edd8f78b405b2b26bdd2aa6e347 into c8e544296fc3436195cd0749990d0cf8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:34,897 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:34,897 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/B, priority=13, startTime=1732742134464; duration=0sec 2024-11-27T21:15:34,898 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:34,898 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:B 2024-11-27T21:15:34,898 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:34,898 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:34,899 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/C is initiating minor compaction (all files) 2024-11-27T21:15:34,899 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/C in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:34,899 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/699f213ab65c416ab00444e508692c1a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/8596a71aa7bd4756a358cf48d1e761e2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e2f371677a8f4cfd87291d3a5edb18c6] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=36.1 K 2024-11-27T21:15:34,899 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 699f213ab65c416ab00444e508692c1a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732742128922 2024-11-27T21:15:34,899 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 8596a71aa7bd4756a358cf48d1e761e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732742131178 2024-11-27T21:15:34,900 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting e2f371677a8f4cfd87291d3a5edb18c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732742133176 2024-11-27T21:15:34,903 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#A#compaction#352 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:34,903 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/e45bd0ad4f354f6eb1c2ccf402604264 is 175, key is test_row_0/A:col10/1732742133201/Put/seqid=0 2024-11-27T21:15:34,910 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#C#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:34,910 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/4c9d765082b8431c897b49f8a63a219c is 50, key is test_row_0/C:col10/1732742133201/Put/seqid=0 2024-11-27T21:15:34,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742245_1421 (size=31699) 2024-11-27T21:15:34,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742246_1422 (size=12745) 2024-11-27T21:15:34,918 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/e45bd0ad4f354f6eb1c2ccf402604264 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/e45bd0ad4f354f6eb1c2ccf402604264 2024-11-27T21:15:34,925 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/4c9d765082b8431c897b49f8a63a219c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/4c9d765082b8431c897b49f8a63a219c 2024-11-27T21:15:34,927 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/A of 49ff3edd8f78b405b2b26bdd2aa6e347 into e45bd0ad4f354f6eb1c2ccf402604264(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:34,927 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:34,927 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/A, priority=13, startTime=1732742134463; duration=0sec 2024-11-27T21:15:34,927 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:34,927 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:A 2024-11-27T21:15:34,931 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/C of 49ff3edd8f78b405b2b26bdd2aa6e347 into 4c9d765082b8431c897b49f8a63a219c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:34,931 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:34,931 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/C, priority=13, startTime=1732742134465; duration=0sec 2024-11-27T21:15:34,931 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:34,931 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:C 2024-11-27T21:15:34,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:34,943 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277bc3ed3a1bb24e12aacfb99cae568614_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277bc3ed3a1bb24e12aacfb99cae568614_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:34,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/6482802e6d1a4e4594db47766bac2332, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:34,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/6482802e6d1a4e4594db47766bac2332 is 175, key is test_row_0/A:col10/1732742133250/Put/seqid=0 2024-11-27T21:15:34,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742247_1423 (size=31255) 2024-11-27T21:15:35,350 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=390, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/6482802e6d1a4e4594db47766bac2332 2024-11-27T21:15:35,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/43629e69896e4452aaca57a4ace518d5 is 50, key is test_row_0/B:col10/1732742133250/Put/seqid=0 2024-11-27T21:15:35,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742248_1424 (size=12301) 2024-11-27T21:15:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:35,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T21:15:35,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742195458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:35,493 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_1' in 49ff3edd8f78b405b2b26bdd2aa6e347 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:15:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:35,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742195491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742195491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742195491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742195493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:35,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742195609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:35,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742195619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:35,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742195619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:35,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742195619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:35,762 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/43629e69896e4452aaca57a4ace518d5 2024-11-27T21:15:35,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/247c846fffd84a34917544512cf4bd73 is 50, key is test_row_0/C:col10/1732742133250/Put/seqid=0 2024-11-27T21:15:35,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742249_1425 (size=12301) 2024-11-27T21:15:35,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742195829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:35,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742195839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:35,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742195839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:35,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742195839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:35,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:35,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:36,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742196138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:36,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742196159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:36,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:36,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742196159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:36,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742196159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:36,173 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/247c846fffd84a34917544512cf4bd73 2024-11-27T21:15:36,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/6482802e6d1a4e4594db47766bac2332 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6482802e6d1a4e4594db47766bac2332 2024-11-27T21:15:36,180 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6482802e6d1a4e4594db47766bac2332, entries=150, sequenceid=390, filesize=30.5 K 2024-11-27T21:15:36,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/43629e69896e4452aaca57a4ace518d5 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/43629e69896e4452aaca57a4ace518d5 2024-11-27T21:15:36,184 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/43629e69896e4452aaca57a4ace518d5, entries=150, sequenceid=390, filesize=12.0 K 2024-11-27T21:15:36,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/247c846fffd84a34917544512cf4bd73 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/247c846fffd84a34917544512cf4bd73 2024-11-27T21:15:36,188 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/247c846fffd84a34917544512cf4bd73, entries=150, sequenceid=390, filesize=12.0 K 2024-11-27T21:15:36,190 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 1664ms, sequenceid=390, compaction requested=false 2024-11-27T21:15:36,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:36,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:36,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-27T21:15:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-27T21:15:36,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-27T21:15:36,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8900 sec 2024-11-27T21:15:36,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.8940 sec 2024-11-27T21:15:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:36,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-27T21:15:36,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:36,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:36,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:36,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:36,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:36,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:36,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112796b9bef34b3b4773bd8aea87ab97b5b3_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742135454/Put/seqid=0 2024-11-27T21:15:36,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742250_1426 (size=14994) 2024-11-27T21:15:36,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742196708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:36,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:36,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742196719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:36,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:36,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742196720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:36,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742196720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:36,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742196828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:36,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742196839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:36,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742196839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742196839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:37,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:37,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742197039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:37,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:37,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742197059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:37,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:37,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742197060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:37,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:37,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742197060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:37,066 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:37,070 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112796b9bef34b3b4773bd8aea87ab97b5b3_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112796b9bef34b3b4773bd8aea87ab97b5b3_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:37,071 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/5908fcb5165242fe969fd317322d5fae, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:37,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/5908fcb5165242fe969fd317322d5fae is 175, key is test_row_0/A:col10/1732742135454/Put/seqid=0 2024-11-27T21:15:37,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742251_1427 (size=39949) 2024-11-27T21:15:37,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742197358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:37,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742197369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:37,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742197369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:37,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742197369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T21:15:37,469 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:37,470 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:15:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-27T21:15:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T21:15:37,471 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:15:37,472 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:15:37,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:15:37,484 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=406, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/5908fcb5165242fe969fd317322d5fae 2024-11-27T21:15:37,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/f6f22bca12544384a2786e9118cbf36f is 50, key is test_row_0/B:col10/1732742135454/Put/seqid=0 2024-11-27T21:15:37,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742252_1428 (size=12301) 2024-11-27T21:15:37,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742197509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:37,520 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_1' in 49ff3edd8f78b405b2b26bdd2aa6e347 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4267 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:15:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:37,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T21:15:37,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T21:15:37,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:37,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:37,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:37,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:37,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:37,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T21:15:37,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:37,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:37,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:37,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:37,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T21:15:37,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742197869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:37,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742197879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:37,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/f6f22bca12544384a2786e9118cbf36f 2024-11-27T21:15:37,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742197888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742197888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:37,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/dfb6d48a22e74f539b7c0a01a2d41564 is 50, key is test_row_0/C:col10/1732742135454/Put/seqid=0 2024-11-27T21:15:37,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742253_1429 (size=12301) 2024-11-27T21:15:37,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T21:15:37,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:37,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:37,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:37,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:37,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:38,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T21:15:38,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:38,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:38,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:38,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:38,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T21:15:38,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T21:15:38,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:38,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:38,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:38,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:38,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:38,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:38,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/dfb6d48a22e74f539b7c0a01a2d41564 2024-11-27T21:15:38,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/5908fcb5165242fe969fd317322d5fae as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/5908fcb5165242fe969fd317322d5fae 2024-11-27T21:15:38,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/5908fcb5165242fe969fd317322d5fae, entries=200, sequenceid=406, filesize=39.0 K 2024-11-27T21:15:38,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/f6f22bca12544384a2786e9118cbf36f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/f6f22bca12544384a2786e9118cbf36f 2024-11-27T21:15:38,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/f6f22bca12544384a2786e9118cbf36f, entries=150, sequenceid=406, filesize=12.0 K 2024-11-27T21:15:38,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/dfb6d48a22e74f539b7c0a01a2d41564 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/dfb6d48a22e74f539b7c0a01a2d41564 2024-11-27T21:15:38,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/dfb6d48a22e74f539b7c0a01a2d41564, entries=150, sequenceid=406, filesize=12.0 K 2024-11-27T21:15:38,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 1668ms, sequenceid=406, compaction requested=true 2024-11-27T21:15:38,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:38,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:15:38,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:38,323 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:38,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:15:38,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:38,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:38,323 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:38,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:38,324 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:38,324 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:38,324 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/B is initiating minor compaction (all files) 2024-11-27T21:15:38,324 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/A is initiating minor compaction (all files) 2024-11-27T21:15:38,324 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/B in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:38,324 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/A in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:38,324 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/c8e544296fc3436195cd0749990d0cf8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/43629e69896e4452aaca57a4ace518d5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/f6f22bca12544384a2786e9118cbf36f] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=36.5 K 2024-11-27T21:15:38,324 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/e45bd0ad4f354f6eb1c2ccf402604264, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6482802e6d1a4e4594db47766bac2332, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/5908fcb5165242fe969fd317322d5fae] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=100.5 K 2024-11-27T21:15:38,324 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:38,324 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/e45bd0ad4f354f6eb1c2ccf402604264, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6482802e6d1a4e4594db47766bac2332, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/5908fcb5165242fe969fd317322d5fae] 2024-11-27T21:15:38,324 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting c8e544296fc3436195cd0749990d0cf8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732742133176 2024-11-27T21:15:38,325 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting e45bd0ad4f354f6eb1c2ccf402604264, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732742133176 2024-11-27T21:15:38,325 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 43629e69896e4452aaca57a4ace518d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732742133233 2024-11-27T21:15:38,325 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6482802e6d1a4e4594db47766bac2332, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732742133233 2024-11-27T21:15:38,325 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting f6f22bca12544384a2786e9118cbf36f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732742135454 2024-11-27T21:15:38,325 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5908fcb5165242fe969fd317322d5fae, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732742135454 2024-11-27T21:15:38,332 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#B#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:38,333 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/1bf1a86ba4a44aa5891d7ea4b0a9fe91 is 50, key is test_row_0/B:col10/1732742135454/Put/seqid=0 2024-11-27T21:15:38,334 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:38,336 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127352a42960588434885b8f54a7be17177_49ff3edd8f78b405b2b26bdd2aa6e347 store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:38,338 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127352a42960588434885b8f54a7be17177_49ff3edd8f78b405b2b26bdd2aa6e347, store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:38,338 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127352a42960588434885b8f54a7be17177_49ff3edd8f78b405b2b26bdd2aa6e347 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:38,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742254_1430 (size=12847) 2024-11-27T21:15:38,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742255_1431 (size=4469) 2024-11-27T21:15:38,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T21:15:38,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:38,389 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T21:15:38,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:38,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:38,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:38,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:38,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:38,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:38,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112747d7928742fa447bb6219610a2d618b0_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742136717/Put/seqid=0 2024-11-27T21:15:38,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742256_1432 (size=12454) 2024-11-27T21:15:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T21:15:38,743 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/1bf1a86ba4a44aa5891d7ea4b0a9fe91 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/1bf1a86ba4a44aa5891d7ea4b0a9fe91 2024-11-27T21:15:38,747 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/B of 49ff3edd8f78b405b2b26bdd2aa6e347 into 1bf1a86ba4a44aa5891d7ea4b0a9fe91(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:38,747 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:38,747 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/B, priority=13, startTime=1732742138323; duration=0sec 2024-11-27T21:15:38,747 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:38,747 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:B 2024-11-27T21:15:38,747 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:38,748 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#A#compaction#362 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:38,748 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:38,749 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/C is initiating minor compaction (all files) 2024-11-27T21:15:38,749 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/9ad7cbb1ce034a499cbfe8a630d0a31b is 175, key is test_row_0/A:col10/1732742135454/Put/seqid=0 2024-11-27T21:15:38,749 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/C in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:38,749 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/4c9d765082b8431c897b49f8a63a219c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/247c846fffd84a34917544512cf4bd73, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/dfb6d48a22e74f539b7c0a01a2d41564] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=36.5 K 2024-11-27T21:15:38,749 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c9d765082b8431c897b49f8a63a219c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732742133176 2024-11-27T21:15:38,749 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 247c846fffd84a34917544512cf4bd73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732742133233 2024-11-27T21:15:38,750 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting dfb6d48a22e74f539b7c0a01a2d41564, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732742135454 2024-11-27T21:15:38,756 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#C#compaction#364 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:38,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742257_1433 (size=31801) 2024-11-27T21:15:38,756 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/82020b4ce82e42df9efac13bd3b1e3d0 is 50, key is test_row_0/C:col10/1732742135454/Put/seqid=0 2024-11-27T21:15:38,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742258_1434 (size=12847) 2024-11-27T21:15:38,762 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/9ad7cbb1ce034a499cbfe8a630d0a31b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9ad7cbb1ce034a499cbfe8a630d0a31b 2024-11-27T21:15:38,766 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/A of 49ff3edd8f78b405b2b26bdd2aa6e347 into 9ad7cbb1ce034a499cbfe8a630d0a31b(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:38,766 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:38,766 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/A, priority=13, startTime=1732742138323; duration=0sec 2024-11-27T21:15:38,766 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:38,766 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:A 2024-11-27T21:15:38,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:38,802 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112747d7928742fa447bb6219610a2d618b0_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112747d7928742fa447bb6219610a2d618b0_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:38,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/0d25de9e960e4a4db166d323a8a06e31, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:38,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/0d25de9e960e4a4db166d323a8a06e31 is 175, key is test_row_0/A:col10/1732742136717/Put/seqid=0 2024-11-27T21:15:38,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742259_1435 (size=31255) 2024-11-27T21:15:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:38,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:38,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:38,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742198927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:38,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:38,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742198929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:38,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:38,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742198929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:38,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:38,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742198930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:39,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742199048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:39,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742199049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:39,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742199049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:39,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742199049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:39,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:39,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:39,166 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/82020b4ce82e42df9efac13bd3b1e3d0 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/82020b4ce82e42df9efac13bd3b1e3d0 2024-11-27T21:15:39,170 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/C of 49ff3edd8f78b405b2b26bdd2aa6e347 into 82020b4ce82e42df9efac13bd3b1e3d0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:39,170 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:39,170 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/C, priority=13, startTime=1732742138323; duration=0sec 2024-11-27T21:15:39,170 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:39,170 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:C 2024-11-27T21:15:39,212 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=425, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/0d25de9e960e4a4db166d323a8a06e31 2024-11-27T21:15:39,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/87524c3845694ef99bfb9327e8806922 is 50, key is test_row_0/B:col10/1732742136717/Put/seqid=0 2024-11-27T21:15:39,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742260_1436 (size=12301) 2024-11-27T21:15:39,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742199258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:39,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742199259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:39,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:39,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742199259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:39,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:39,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:39,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742199259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:39,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:39,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:39,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742199579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:39,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742199579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:39,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742199579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:39,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742199580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T21:15:39,623 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/87524c3845694ef99bfb9327e8806922 2024-11-27T21:15:39,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/232d5ebec1ec42208d72139cafa64815 is 50, key is test_row_0/C:col10/1732742136717/Put/seqid=0 2024-11-27T21:15:39,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742261_1437 (size=12301) 2024-11-27T21:15:40,034 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/232d5ebec1ec42208d72139cafa64815 2024-11-27T21:15:40,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/0d25de9e960e4a4db166d323a8a06e31 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/0d25de9e960e4a4db166d323a8a06e31 2024-11-27T21:15:40,042 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/0d25de9e960e4a4db166d323a8a06e31, entries=150, sequenceid=425, filesize=30.5 K 2024-11-27T21:15:40,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/87524c3845694ef99bfb9327e8806922 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/87524c3845694ef99bfb9327e8806922 2024-11-27T21:15:40,046 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/87524c3845694ef99bfb9327e8806922, entries=150, sequenceid=425, filesize=12.0 K 2024-11-27T21:15:40,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/232d5ebec1ec42208d72139cafa64815 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/232d5ebec1ec42208d72139cafa64815 2024-11-27T21:15:40,050 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/232d5ebec1ec42208d72139cafa64815, entries=150, sequenceid=425, filesize=12.0 K 2024-11-27T21:15:40,051 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 1661ms, sequenceid=425, compaction requested=false 2024-11-27T21:15:40,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:40,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:40,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-27T21:15:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-27T21:15:40,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-27T21:15:40,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5800 sec 2024-11-27T21:15:40,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.5840 sec 2024-11-27T21:15:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:40,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T21:15:40,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:40,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:40,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:40,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:40,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:40,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:40,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b41c55f581cd4b4092a93b4c21b76f0c_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742140101/Put/seqid=0 2024-11-27T21:15:40,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742262_1438 (size=14994) 2024-11-27T21:15:40,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742200141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:40,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742200142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:40,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742200142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:40,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742200148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:40,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:40,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742200259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:40,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:40,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742200259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:40,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742200259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:40,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:40,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:40,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742200269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:40,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:40,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742200469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:40,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742200478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742200479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:40,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742200489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:40,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:40,503 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T21:15:40,516 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:40,520 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b41c55f581cd4b4092a93b4c21b76f0c_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b41c55f581cd4b4092a93b4c21b76f0c_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:40,520 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/db7baac32a4b4497ab00c25755e995fb, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:40,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/db7baac32a4b4497ab00c25755e995fb is 175, key is test_row_0/A:col10/1732742140101/Put/seqid=0 2024-11-27T21:15:40,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742263_1439 (size=39949) 2024-11-27T21:15:40,525 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=442, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/db7baac32a4b4497ab00c25755e995fb 2024-11-27T21:15:40,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/3665938f2c3946fab50445bbcb5fea19 is 50, key is test_row_0/B:col10/1732742140101/Put/seqid=0 2024-11-27T21:15:40,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742264_1440 (size=12301) 2024-11-27T21:15:40,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/3665938f2c3946fab50445bbcb5fea19 2024-11-27T21:15:40,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/5228391e041d46c39932c760357d9ab4 is 50, key is test_row_0/C:col10/1732742140101/Put/seqid=0 2024-11-27T21:15:40,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742265_1441 (size=12301) 2024-11-27T21:15:40,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742200789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:40,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742200798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:40,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742200798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:40,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:40,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742200809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:40,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:40,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:40,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/5228391e041d46c39932c760357d9ab4 2024-11-27T21:15:40,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/db7baac32a4b4497ab00c25755e995fb as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/db7baac32a4b4497ab00c25755e995fb 2024-11-27T21:15:40,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/db7baac32a4b4497ab00c25755e995fb, entries=200, sequenceid=442, filesize=39.0 K 2024-11-27T21:15:40,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/3665938f2c3946fab50445bbcb5fea19 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/3665938f2c3946fab50445bbcb5fea19 2024-11-27T21:15:40,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/3665938f2c3946fab50445bbcb5fea19, entries=150, sequenceid=442, filesize=12.0 K 2024-11-27T21:15:40,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/5228391e041d46c39932c760357d9ab4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/5228391e041d46c39932c760357d9ab4 2024-11-27T21:15:40,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/5228391e041d46c39932c760357d9ab4, entries=150, sequenceid=442, filesize=12.0 K 2024-11-27T21:15:40,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 862ms, sequenceid=442, compaction requested=true 2024-11-27T21:15:40,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:40,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:15:40,966 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:40,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:40,966 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:40,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:15:40,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:40,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49ff3edd8f78b405b2b26bdd2aa6e347:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:15:40,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:40,967 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:40,967 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:40,967 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/B is initiating minor compaction (all files) 2024-11-27T21:15:40,967 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/A is initiating minor compaction (all files) 2024-11-27T21:15:40,967 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/A in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:40,967 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/B in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:40,967 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9ad7cbb1ce034a499cbfe8a630d0a31b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/0d25de9e960e4a4db166d323a8a06e31, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/db7baac32a4b4497ab00c25755e995fb] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=100.6 K 2024-11-27T21:15:40,967 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/1bf1a86ba4a44aa5891d7ea4b0a9fe91, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/87524c3845694ef99bfb9327e8806922, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/3665938f2c3946fab50445bbcb5fea19] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=36.6 K 2024-11-27T21:15:40,967 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:40,967 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9ad7cbb1ce034a499cbfe8a630d0a31b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/0d25de9e960e4a4db166d323a8a06e31, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/db7baac32a4b4497ab00c25755e995fb] 2024-11-27T21:15:40,967 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bf1a86ba4a44aa5891d7ea4b0a9fe91, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732742135454 2024-11-27T21:15:40,968 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ad7cbb1ce034a499cbfe8a630d0a31b, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732742135454 2024-11-27T21:15:40,968 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 87524c3845694ef99bfb9327e8806922, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732742136717 2024-11-27T21:15:40,968 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d25de9e960e4a4db166d323a8a06e31, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732742136717 2024-11-27T21:15:40,968 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting db7baac32a4b4497ab00c25755e995fb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1732742138925 2024-11-27T21:15:40,969 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 3665938f2c3946fab50445bbcb5fea19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1732742138925 2024-11-27T21:15:40,975 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:40,976 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112715466977d6a0490ea9ac91996cf68e5f_49ff3edd8f78b405b2b26bdd2aa6e347 store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:40,976 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#B#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:40,977 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/dce287c8b6b146d5a0d979cc2049b20b is 50, key is test_row_0/B:col10/1732742140101/Put/seqid=0 2024-11-27T21:15:40,977 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112715466977d6a0490ea9ac91996cf68e5f_49ff3edd8f78b405b2b26bdd2aa6e347, store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:40,978 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112715466977d6a0490ea9ac91996cf68e5f_49ff3edd8f78b405b2b26bdd2aa6e347 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:40,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742266_1442 (size=12949) 2024-11-27T21:15:40,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742267_1443 (size=4469) 2024-11-27T21:15:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:41,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T21:15:41,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:41,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:41,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:41,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:41,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:41,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:41,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271424c4eb3199451dbb90337724913bd4_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742141300/Put/seqid=0 2024-11-27T21:15:41,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742268_1444 (size=14994) 2024-11-27T21:15:41,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742201366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:41,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742201366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:41,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742201366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:41,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742201367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:41,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:41,384 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#A#compaction#370 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:41,385 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/6ea7dd5754be43f8b8ac735484f48839 is 175, key is test_row_0/A:col10/1732742140101/Put/seqid=0 2024-11-27T21:15:41,388 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/dce287c8b6b146d5a0d979cc2049b20b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/dce287c8b6b146d5a0d979cc2049b20b 2024-11-27T21:15:41,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742269_1445 (size=31903) 2024-11-27T21:15:41,392 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/B of 49ff3edd8f78b405b2b26bdd2aa6e347 into dce287c8b6b146d5a0d979cc2049b20b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:41,392 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:41,392 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/B, priority=13, startTime=1732742140966; duration=0sec 2024-11-27T21:15:41,392 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:15:41,392 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:B 2024-11-27T21:15:41,392 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:15:41,394 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:15:41,394 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): 49ff3edd8f78b405b2b26bdd2aa6e347/C is initiating minor compaction (all files) 2024-11-27T21:15:41,394 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49ff3edd8f78b405b2b26bdd2aa6e347/C in TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:41,394 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/82020b4ce82e42df9efac13bd3b1e3d0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/232d5ebec1ec42208d72139cafa64815, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/5228391e041d46c39932c760357d9ab4] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp, totalSize=36.6 K 2024-11-27T21:15:41,394 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 82020b4ce82e42df9efac13bd3b1e3d0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732742135454 2024-11-27T21:15:41,394 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 232d5ebec1ec42208d72139cafa64815, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732742136717 2024-11-27T21:15:41,395 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 5228391e041d46c39932c760357d9ab4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1732742138925 2024-11-27T21:15:41,401 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49ff3edd8f78b405b2b26bdd2aa6e347#C#compaction#373 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:15:41,401 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/72a05f9c25e14aa7a2e06f7f990a178f is 50, key is test_row_0/C:col10/1732742140101/Put/seqid=0 2024-11-27T21:15:41,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742270_1446 (size=12949) 2024-11-27T21:15:41,408 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/72a05f9c25e14aa7a2e06f7f990a178f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/72a05f9c25e14aa7a2e06f7f990a178f 2024-11-27T21:15:41,413 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/C of 49ff3edd8f78b405b2b26bdd2aa6e347 into 72a05f9c25e14aa7a2e06f7f990a178f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:41,413 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:41,413 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/C, priority=13, startTime=1732742140966; duration=0sec 2024-11-27T21:15:41,413 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:41,413 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:C 2024-11-27T21:15:41,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742201488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:41,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742201489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:41,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742201489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:41,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742201489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:41,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35956 deadline: 1732742201548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:41,556 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_1' in 49ff3edd8f78b405b2b26bdd2aa6e347 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8303 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:15:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T21:15:41,629 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:41,630 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:15:41,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-27T21:15:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T21:15:41,632 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:15:41,632 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:15:41,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:15:41,682 INFO [Thread-1661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1661. 2024-11-27T21:15:41,682 DEBUG [Thread-1661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:41,682 DEBUG [Thread-1661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:41,682 DEBUG [Thread-1661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:41,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:41,687 INFO [Thread-1665 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1665. 2024-11-27T21:15:41,687 DEBUG [Thread-1665 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:41,687 DEBUG [Thread-1665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:41,687 DEBUG [Thread-1665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:41,687 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:41,693 INFO [Thread-1663 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1663. 2024-11-27T21:15:41,693 DEBUG [Thread-1663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:41,693 DEBUG [Thread-1663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:41,693 DEBUG [Thread-1663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:41,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:41,694 INFO [Thread-1667 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1667. 2024-11-27T21:15:41,694 DEBUG [Thread-1667 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:41,694 DEBUG [Thread-1667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:41,694 DEBUG [Thread-1667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:41,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:41,697 INFO [Thread-1669 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1669. 2024-11-27T21:15:41,697 DEBUG [Thread-1669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:41,697 DEBUG [Thread-1669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:41,697 DEBUG [Thread-1669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:41,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:41,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:41,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:41,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742201699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742201699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742201699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:41,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742201699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:41,720 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:41,723 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271424c4eb3199451dbb90337724913bd4_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271424c4eb3199451dbb90337724913bd4_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:41,723 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/9f82bee97c544e28b305e7a47cc87081, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:41,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/9f82bee97c544e28b305e7a47cc87081 is 175, key is test_row_0/A:col10/1732742141300/Put/seqid=0 2024-11-27T21:15:41,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742271_1447 (size=39949) 2024-11-27T21:15:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T21:15:41,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T21:15:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:41,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:41,792 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/6ea7dd5754be43f8b8ac735484f48839 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6ea7dd5754be43f8b8ac735484f48839 2024-11-27T21:15:41,796 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49ff3edd8f78b405b2b26bdd2aa6e347/A of 49ff3edd8f78b405b2b26bdd2aa6e347 into 6ea7dd5754be43f8b8ac735484f48839(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:15:41,796 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:41,796 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., storeName=49ff3edd8f78b405b2b26bdd2aa6e347/A, priority=13, startTime=1732742140966; duration=0sec 2024-11-27T21:15:41,796 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:15:41,796 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49ff3edd8f78b405b2b26bdd2aa6e347:A 2024-11-27T21:15:41,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T21:15:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:41,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T21:15:42,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:42,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:42,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742202010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742202010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742202010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:42,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742202010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:42,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T21:15:42,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:42,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:42,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:42,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,128 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=461, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/9f82bee97c544e28b305e7a47cc87081 2024-11-27T21:15:42,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/8bd38044e9f8442dbbdaebbe38f8aff6 is 50, key is test_row_0/B:col10/1732742141300/Put/seqid=0 2024-11-27T21:15:42,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742272_1448 (size=12301) 2024-11-27T21:15:42,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T21:15:42,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:42,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:42,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:42,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T21:15:42,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T21:15:42,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:42,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:42,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:42,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:42,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:42,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35926 deadline: 1732742202519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35968 deadline: 1732742202519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35980 deadline: 1732742202519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:42,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35972 deadline: 1732742202519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:42,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:42,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:42,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:15:42,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/8bd38044e9f8442dbbdaebbe38f8aff6 2024-11-27T21:15:42,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/741c3000aa9a4f3aa7ae8366d4ec290e is 50, key is test_row_0/C:col10/1732742141300/Put/seqid=0 2024-11-27T21:15:42,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742273_1449 (size=12301) 2024-11-27T21:15:42,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T21:15:42,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:42,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:42,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:42,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T21:15:42,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:42,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:42,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:42,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T21:15:42,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T21:15:42,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:42,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:42,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:42,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:42,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/741c3000aa9a4f3aa7ae8366d4ec290e 2024-11-27T21:15:42,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/9f82bee97c544e28b305e7a47cc87081 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9f82bee97c544e28b305e7a47cc87081 2024-11-27T21:15:42,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9f82bee97c544e28b305e7a47cc87081, entries=200, sequenceid=461, filesize=39.0 K 2024-11-27T21:15:42,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/8bd38044e9f8442dbbdaebbe38f8aff6 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/8bd38044e9f8442dbbdaebbe38f8aff6 2024-11-27T21:15:42,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/8bd38044e9f8442dbbdaebbe38f8aff6, entries=150, sequenceid=461, filesize=12.0 K 2024-11-27T21:15:42,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/741c3000aa9a4f3aa7ae8366d4ec290e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/741c3000aa9a4f3aa7ae8366d4ec290e 2024-11-27T21:15:42,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/741c3000aa9a4f3aa7ae8366d4ec290e, entries=150, sequenceid=461, filesize=12.0 K 2024-11-27T21:15:42,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 1661ms, sequenceid=461, compaction requested=false 2024-11-27T21:15:42,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:43,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T21:15:43,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:43,010 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-27T21:15:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:43,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b9d68b7ae0664d6dbbd2216acb814946_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742141365/Put/seqid=0 2024-11-27T21:15:43,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742274_1450 (size=12454) 2024-11-27T21:15:43,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:43,422 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b9d68b7ae0664d6dbbd2216acb814946_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b9d68b7ae0664d6dbbd2216acb814946_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:43,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/abeb51be13a64e8295531df3fc6d503e, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:43,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/abeb51be13a64e8295531df3fc6d503e is 175, key is test_row_0/A:col10/1732742141365/Put/seqid=0 2024-11-27T21:15:43,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742275_1451 (size=31255) 2024-11-27T21:15:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:43,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. as already flushing 2024-11-27T21:15:43,558 INFO [Thread-1656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1656. 2024-11-27T21:15:43,558 INFO [Thread-1652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1652. 2024-11-27T21:15:43,558 INFO [Thread-1658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1658. 2024-11-27T21:15:43,558 DEBUG [Thread-1656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:43,558 DEBUG [Thread-1656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:43,558 DEBUG [Thread-1652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:43,558 DEBUG [Thread-1658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:43,558 DEBUG [Thread-1652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:43,558 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:43,558 DEBUG [Thread-1656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:43,558 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:43,558 DEBUG [Thread-1652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:43,558 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:43,558 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:43,558 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:43,559 INFO [Thread-1654 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1654. 2024-11-27T21:15:43,559 DEBUG [Thread-1654 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:43,559 DEBUG [Thread-1654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:43,559 DEBUG [Thread-1654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:43,559 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T21:15:43,827 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=477, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/abeb51be13a64e8295531df3fc6d503e 2024-11-27T21:15:43,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/ed5ce6c928564b7b950434a7e27f78e3 is 50, key is test_row_0/B:col10/1732742141365/Put/seqid=0 2024-11-27T21:15:43,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742276_1452 (size=12301) 2024-11-27T21:15:44,237 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/ed5ce6c928564b7b950434a7e27f78e3 2024-11-27T21:15:44,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/e0110edcdcbd4b26a6538630cc67b4f2 is 50, key is test_row_0/C:col10/1732742141365/Put/seqid=0 2024-11-27T21:15:44,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742277_1453 (size=12301) 2024-11-27T21:15:44,647 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/e0110edcdcbd4b26a6538630cc67b4f2 2024-11-27T21:15:44,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/abeb51be13a64e8295531df3fc6d503e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/abeb51be13a64e8295531df3fc6d503e 2024-11-27T21:15:44,654 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/abeb51be13a64e8295531df3fc6d503e, entries=150, sequenceid=477, filesize=30.5 K 2024-11-27T21:15:44,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/ed5ce6c928564b7b950434a7e27f78e3 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/ed5ce6c928564b7b950434a7e27f78e3 2024-11-27T21:15:44,658 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/ed5ce6c928564b7b950434a7e27f78e3, entries=150, sequenceid=477, filesize=12.0 K 2024-11-27T21:15:44,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/e0110edcdcbd4b26a6538630cc67b4f2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e0110edcdcbd4b26a6538630cc67b4f2 2024-11-27T21:15:44,661 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e0110edcdcbd4b26a6538630cc67b4f2, entries=150, sequenceid=477, filesize=12.0 K 2024-11-27T21:15:44,662 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 1652ms, sequenceid=477, compaction requested=true 2024-11-27T21:15:44,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 49ff3edd8f78b405b2b26bdd2aa6e347: 2024-11-27T21:15:44,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:44,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-27T21:15:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-27T21:15:44,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-27T21:15:44,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0310 sec 2024-11-27T21:15:44,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 3.0340 sec 2024-11-27T21:15:45,314 WARN [regionserver/4585c39bed03:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-27T21:15:45,314 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/98262bcf170e423c938623c5e1e597e5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/3f8c8f98f69744dfa25a7fb3110d8c87, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/cfd5f8f8a0c84552b26f6683685f673f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/4e5e608945c24b729b0423e3be1dedf8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/a56d71b2b7e34c3a987efb7dc7cfbad0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/29176966dcea429f84882f5cf3d82633, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/3c01f32c4147429aa82549dcaa894a75, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/f42fa8b58a9c4d40abe89bb2c1eb7c2e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9fef0041ecac48b2b5a8581922fbcb70, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/8df47496313a4ee3be45b6db260e07cf, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/e5331d2b245f4394980cf3f165d24847, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6a80bb0e228344cf85afd62f6c2a9d0b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/171064c00e70477bbb98a2df634cdc28, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/e45bd0ad4f354f6eb1c2ccf402604264, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6482802e6d1a4e4594db47766bac2332, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/5908fcb5165242fe969fd317322d5fae, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9ad7cbb1ce034a499cbfe8a630d0a31b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/0d25de9e960e4a4db166d323a8a06e31, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/db7baac32a4b4497ab00c25755e995fb] to archive 2024-11-27T21:15:45,315 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:15:45,317 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/98262bcf170e423c938623c5e1e597e5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/98262bcf170e423c938623c5e1e597e5 2024-11-27T21:15:45,318 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/3f8c8f98f69744dfa25a7fb3110d8c87 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/3f8c8f98f69744dfa25a7fb3110d8c87 2024-11-27T21:15:45,318 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/cfd5f8f8a0c84552b26f6683685f673f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/cfd5f8f8a0c84552b26f6683685f673f 2024-11-27T21:15:45,319 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/4e5e608945c24b729b0423e3be1dedf8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/4e5e608945c24b729b0423e3be1dedf8 2024-11-27T21:15:45,320 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/a56d71b2b7e34c3a987efb7dc7cfbad0 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/a56d71b2b7e34c3a987efb7dc7cfbad0 2024-11-27T21:15:45,321 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/29176966dcea429f84882f5cf3d82633 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/29176966dcea429f84882f5cf3d82633 2024-11-27T21:15:45,322 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/3c01f32c4147429aa82549dcaa894a75 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/3c01f32c4147429aa82549dcaa894a75 2024-11-27T21:15:45,323 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/f42fa8b58a9c4d40abe89bb2c1eb7c2e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/f42fa8b58a9c4d40abe89bb2c1eb7c2e 2024-11-27T21:15:45,323 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9fef0041ecac48b2b5a8581922fbcb70 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9fef0041ecac48b2b5a8581922fbcb70 2024-11-27T21:15:45,324 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/8df47496313a4ee3be45b6db260e07cf to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/8df47496313a4ee3be45b6db260e07cf 2024-11-27T21:15:45,325 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/e5331d2b245f4394980cf3f165d24847 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/e5331d2b245f4394980cf3f165d24847 2024-11-27T21:15:45,326 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6a80bb0e228344cf85afd62f6c2a9d0b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6a80bb0e228344cf85afd62f6c2a9d0b 2024-11-27T21:15:45,327 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/171064c00e70477bbb98a2df634cdc28 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/171064c00e70477bbb98a2df634cdc28 2024-11-27T21:15:45,328 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/e45bd0ad4f354f6eb1c2ccf402604264 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/e45bd0ad4f354f6eb1c2ccf402604264 2024-11-27T21:15:45,329 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6482802e6d1a4e4594db47766bac2332 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6482802e6d1a4e4594db47766bac2332 2024-11-27T21:15:45,329 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/5908fcb5165242fe969fd317322d5fae to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/5908fcb5165242fe969fd317322d5fae 2024-11-27T21:15:45,330 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9ad7cbb1ce034a499cbfe8a630d0a31b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9ad7cbb1ce034a499cbfe8a630d0a31b 2024-11-27T21:15:45,331 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/0d25de9e960e4a4db166d323a8a06e31 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/0d25de9e960e4a4db166d323a8a06e31 2024-11-27T21:15:45,332 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/db7baac32a4b4497ab00c25755e995fb to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/db7baac32a4b4497ab00c25755e995fb 2024-11-27T21:15:45,335 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/27169f1c17a242febb925744c6080c87, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/a60b14a921114207be9583eef51647df, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/15253010c6ac4f0a8639a4165ca79b3f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/6f8a39157fc049039a8ad8c8393479f2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/227ecad780054600be37686f717eecbc, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/eed92e63c69c48d680d50ab6905e5dad, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/34493e801642465ba24aef70f5ca1f6e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/63ff1960edd44d9c804a178318b04b1f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/feb968b357af457090edf36bc388cfe2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/bc027f815cce4bee86b15c87607b6e03, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/15fe4cc06a0a4a7784e1f2db3d0f1bc1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/bced16ecab504b6fb62a3d4ccfff4b75, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/c8e544296fc3436195cd0749990d0cf8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/7bad767d277049edb219e6afdd68424d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/43629e69896e4452aaca57a4ace518d5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/1bf1a86ba4a44aa5891d7ea4b0a9fe91, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/f6f22bca12544384a2786e9118cbf36f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/87524c3845694ef99bfb9327e8806922, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/3665938f2c3946fab50445bbcb5fea19] to archive 2024-11-27T21:15:45,336 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:15:45,337 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/27169f1c17a242febb925744c6080c87 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/27169f1c17a242febb925744c6080c87 2024-11-27T21:15:45,337 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/a60b14a921114207be9583eef51647df to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/a60b14a921114207be9583eef51647df 2024-11-27T21:15:45,338 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/15253010c6ac4f0a8639a4165ca79b3f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/15253010c6ac4f0a8639a4165ca79b3f 2024-11-27T21:15:45,339 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/6f8a39157fc049039a8ad8c8393479f2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/6f8a39157fc049039a8ad8c8393479f2 2024-11-27T21:15:45,340 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/227ecad780054600be37686f717eecbc to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/227ecad780054600be37686f717eecbc 2024-11-27T21:15:45,341 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/eed92e63c69c48d680d50ab6905e5dad to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/eed92e63c69c48d680d50ab6905e5dad 2024-11-27T21:15:45,341 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/34493e801642465ba24aef70f5ca1f6e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/34493e801642465ba24aef70f5ca1f6e 2024-11-27T21:15:45,342 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/63ff1960edd44d9c804a178318b04b1f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/63ff1960edd44d9c804a178318b04b1f 2024-11-27T21:15:45,343 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/feb968b357af457090edf36bc388cfe2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/feb968b357af457090edf36bc388cfe2 2024-11-27T21:15:45,344 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/bc027f815cce4bee86b15c87607b6e03 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/bc027f815cce4bee86b15c87607b6e03 2024-11-27T21:15:45,344 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/15fe4cc06a0a4a7784e1f2db3d0f1bc1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/15fe4cc06a0a4a7784e1f2db3d0f1bc1 2024-11-27T21:15:45,345 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/bced16ecab504b6fb62a3d4ccfff4b75 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/bced16ecab504b6fb62a3d4ccfff4b75 2024-11-27T21:15:45,346 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/c8e544296fc3436195cd0749990d0cf8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/c8e544296fc3436195cd0749990d0cf8 2024-11-27T21:15:45,347 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/7bad767d277049edb219e6afdd68424d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/7bad767d277049edb219e6afdd68424d 2024-11-27T21:15:45,348 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/43629e69896e4452aaca57a4ace518d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/43629e69896e4452aaca57a4ace518d5 2024-11-27T21:15:45,348 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/1bf1a86ba4a44aa5891d7ea4b0a9fe91 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/1bf1a86ba4a44aa5891d7ea4b0a9fe91 2024-11-27T21:15:45,349 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/f6f22bca12544384a2786e9118cbf36f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/f6f22bca12544384a2786e9118cbf36f 2024-11-27T21:15:45,350 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/87524c3845694ef99bfb9327e8806922 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/87524c3845694ef99bfb9327e8806922 2024-11-27T21:15:45,351 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/3665938f2c3946fab50445bbcb5fea19 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/3665938f2c3946fab50445bbcb5fea19 2024-11-27T21:15:45,353 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e8e02b98566a4947b2608bf12dbcfe2c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/c25a2f0cd5484f2c9a95a0403d202154, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/12f1571f7c754a9c9c801f510857c05c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/680f7d1b85c54169933d10449bbf80ec, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/33d467ec1e8e4658a938dea45324f3cb, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/ae73a1d7178a4827b9a71216e4ded7db, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/157e7cedf5dc41f098633b0cd753062e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/71ca25a6d1ac4a7f973bc9bdbeb070a3, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/9fa1133c50024445b3d26c61bead74c1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/699f213ab65c416ab00444e508692c1a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e4cf5b05a22d40bd92377d9a337ad8f2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/8596a71aa7bd4756a358cf48d1e761e2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/4c9d765082b8431c897b49f8a63a219c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e2f371677a8f4cfd87291d3a5edb18c6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/247c846fffd84a34917544512cf4bd73, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/82020b4ce82e42df9efac13bd3b1e3d0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/dfb6d48a22e74f539b7c0a01a2d41564, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/232d5ebec1ec42208d72139cafa64815, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/5228391e041d46c39932c760357d9ab4] to archive 2024-11-27T21:15:45,354 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:15:45,354 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e8e02b98566a4947b2608bf12dbcfe2c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e8e02b98566a4947b2608bf12dbcfe2c 2024-11-27T21:15:45,355 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/c25a2f0cd5484f2c9a95a0403d202154 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/c25a2f0cd5484f2c9a95a0403d202154 2024-11-27T21:15:45,356 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/12f1571f7c754a9c9c801f510857c05c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/12f1571f7c754a9c9c801f510857c05c 2024-11-27T21:15:45,357 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/680f7d1b85c54169933d10449bbf80ec to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/680f7d1b85c54169933d10449bbf80ec 2024-11-27T21:15:45,357 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/33d467ec1e8e4658a938dea45324f3cb to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/33d467ec1e8e4658a938dea45324f3cb 2024-11-27T21:15:45,358 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/ae73a1d7178a4827b9a71216e4ded7db to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/ae73a1d7178a4827b9a71216e4ded7db 2024-11-27T21:15:45,359 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/157e7cedf5dc41f098633b0cd753062e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/157e7cedf5dc41f098633b0cd753062e 2024-11-27T21:15:45,360 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/71ca25a6d1ac4a7f973bc9bdbeb070a3 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/71ca25a6d1ac4a7f973bc9bdbeb070a3 2024-11-27T21:15:45,361 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/9fa1133c50024445b3d26c61bead74c1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/9fa1133c50024445b3d26c61bead74c1 2024-11-27T21:15:45,361 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/699f213ab65c416ab00444e508692c1a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/699f213ab65c416ab00444e508692c1a 2024-11-27T21:15:45,362 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e4cf5b05a22d40bd92377d9a337ad8f2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e4cf5b05a22d40bd92377d9a337ad8f2 2024-11-27T21:15:45,363 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/8596a71aa7bd4756a358cf48d1e761e2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/8596a71aa7bd4756a358cf48d1e761e2 2024-11-27T21:15:45,364 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/4c9d765082b8431c897b49f8a63a219c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/4c9d765082b8431c897b49f8a63a219c 2024-11-27T21:15:45,364 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e2f371677a8f4cfd87291d3a5edb18c6 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e2f371677a8f4cfd87291d3a5edb18c6 2024-11-27T21:15:45,365 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/247c846fffd84a34917544512cf4bd73 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/247c846fffd84a34917544512cf4bd73 2024-11-27T21:15:45,366 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/82020b4ce82e42df9efac13bd3b1e3d0 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/82020b4ce82e42df9efac13bd3b1e3d0 2024-11-27T21:15:45,366 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/dfb6d48a22e74f539b7c0a01a2d41564 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/dfb6d48a22e74f539b7c0a01a2d41564 2024-11-27T21:15:45,367 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/232d5ebec1ec42208d72139cafa64815 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/232d5ebec1ec42208d72139cafa64815 2024-11-27T21:15:45,368 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4585c39bed03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/5228391e041d46c39932c760357d9ab4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/5228391e041d46c39932c760357d9ab4 2024-11-27T21:15:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T21:15:45,799 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:51,610 INFO [Thread-1650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1650. 2024-11-27T21:15:51,610 DEBUG [Thread-1650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:15:51,610 DEBUG [Thread-1650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:51,611 DEBUG [Thread-1650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:51,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 964 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2880 rows 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 961 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2882 rows 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 966 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2897 rows 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 967 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2894 rows 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 967 2024-11-27T21:15:51,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2892 rows 2024-11-27T21:15:51,611 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T21:15:51,611 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:15:51,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:51,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:51,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:15:51,612 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T21:15:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T21:15:51,615 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742151614"}]},"ts":"1732742151614"} 2024-11-27T21:15:51,616 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T21:15:51,616 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T21:15:51,616 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:15:51,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49ff3edd8f78b405b2b26bdd2aa6e347, UNASSIGN}] 2024-11-27T21:15:51,618 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49ff3edd8f78b405b2b26bdd2aa6e347, UNASSIGN 2024-11-27T21:15:51,618 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=49ff3edd8f78b405b2b26bdd2aa6e347, regionState=CLOSING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:15:51,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49ff3edd8f78b405b2b26bdd2aa6e347, UNASSIGN because future has completed 2024-11-27T21:15:51,620 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:15:51,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure 49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:15:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T21:15:51,772 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(122): Close 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:51,772 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:15:51,772 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1722): Closing 49ff3edd8f78b405b2b26bdd2aa6e347, disabling compactions & flushes 2024-11-27T21:15:51,772 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:51,772 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:51,772 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. after waiting 0 ms 2024-11-27T21:15:51,772 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:51,772 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(2902): Flushing 49ff3edd8f78b405b2b26bdd2aa6e347 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-27T21:15:51,772 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=A 2024-11-27T21:15:51,772 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:51,772 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=B 2024-11-27T21:15:51,772 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:51,773 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49ff3edd8f78b405b2b26bdd2aa6e347, store=C 2024-11-27T21:15:51,773 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:51,777 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272053eaf28e9f4d0a85c0afccf7b14302_49ff3edd8f78b405b2b26bdd2aa6e347 is 50, key is test_row_0/A:col10/1732742143557/Put/seqid=0 2024-11-27T21:15:51,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742278_1454 (size=12454) 2024-11-27T21:15:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T21:15:52,181 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:52,185 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272053eaf28e9f4d0a85c0afccf7b14302_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272053eaf28e9f4d0a85c0afccf7b14302_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:52,185 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/a12eee94550247b5895db3267abc5256, store: [table=TestAcidGuarantees family=A region=49ff3edd8f78b405b2b26bdd2aa6e347] 2024-11-27T21:15:52,186 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/a12eee94550247b5895db3267abc5256 is 175, key is test_row_0/A:col10/1732742143557/Put/seqid=0 2024-11-27T21:15:52,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742279_1455 (size=31255) 2024-11-27T21:15:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T21:15:52,590 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=485, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/a12eee94550247b5895db3267abc5256 2024-11-27T21:15:52,595 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/c20e3ebbe50a4cfe89228b8d30a4a9be is 50, key is test_row_0/B:col10/1732742143557/Put/seqid=0 2024-11-27T21:15:52,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742280_1456 (size=12301) 2024-11-27T21:15:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T21:15:52,999 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/c20e3ebbe50a4cfe89228b8d30a4a9be 2024-11-27T21:15:53,005 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/6cff3376ce7142eb8c3b59bc37c6b154 is 50, key is test_row_0/C:col10/1732742143557/Put/seqid=0 2024-11-27T21:15:53,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742281_1457 (size=12301) 2024-11-27T21:15:53,409 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/6cff3376ce7142eb8c3b59bc37c6b154 2024-11-27T21:15:53,412 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/A/a12eee94550247b5895db3267abc5256 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/a12eee94550247b5895db3267abc5256 2024-11-27T21:15:53,415 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/a12eee94550247b5895db3267abc5256, entries=150, sequenceid=485, filesize=30.5 K 2024-11-27T21:15:53,416 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/B/c20e3ebbe50a4cfe89228b8d30a4a9be as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/c20e3ebbe50a4cfe89228b8d30a4a9be 2024-11-27T21:15:53,419 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/c20e3ebbe50a4cfe89228b8d30a4a9be, entries=150, sequenceid=485, filesize=12.0 K 2024-11-27T21:15:53,419 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/.tmp/C/6cff3376ce7142eb8c3b59bc37c6b154 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/6cff3376ce7142eb8c3b59bc37c6b154 2024-11-27T21:15:53,422 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/6cff3376ce7142eb8c3b59bc37c6b154, entries=150, sequenceid=485, filesize=12.0 K 2024-11-27T21:15:53,423 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 1651ms, sequenceid=485, compaction requested=true 2024-11-27T21:15:53,426 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/recovered.edits/488.seqid, newMaxSeqId=488, maxSeqId=4 2024-11-27T21:15:53,427 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. 2024-11-27T21:15:53,427 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1676): Region close journal for 49ff3edd8f78b405b2b26bdd2aa6e347: Waiting for close lock at 1732742151772Running coprocessor pre-close hooks at 1732742151772Disabling compacts and flushes for region at 1732742151772Disabling writes for close at 1732742151772Obtaining lock to block concurrent updates at 1732742151772Preparing flush snapshotting stores in 49ff3edd8f78b405b2b26bdd2aa6e347 at 1732742151772Finished memstore snapshotting TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732742151773 (+1 ms)Flushing stores of TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347. at 1732742151773Flushing 49ff3edd8f78b405b2b26bdd2aa6e347/A: creating writer at 1732742151773Flushing mob file 49ff3edd8f78b405b2b26bdd2aa6e347/A: appending metadata at 1732742151777 (+4 ms)Flushing mob file 49ff3edd8f78b405b2b26bdd2aa6e347/A: closing flushed file at 1732742151777Flushing 49ff3edd8f78b405b2b26bdd2aa6e347/A: appending metadata at 1732742152185 (+408 ms)Flushing 49ff3edd8f78b405b2b26bdd2aa6e347/A: closing flushed file at 1732742152186 (+1 ms)Flushing 49ff3edd8f78b405b2b26bdd2aa6e347/B: creating writer at 1732742152593 (+407 ms)Flushing 49ff3edd8f78b405b2b26bdd2aa6e347/B: appending metadata at 1732742152595 (+2 ms)Flushing 49ff3edd8f78b405b2b26bdd2aa6e347/B: closing flushed file at 1732742152595Flushing 49ff3edd8f78b405b2b26bdd2aa6e347/C: creating writer at 1732742153002 (+407 ms)Flushing 49ff3edd8f78b405b2b26bdd2aa6e347/C: appending metadata at 1732742153004 (+2 ms)Flushing 49ff3edd8f78b405b2b26bdd2aa6e347/C: closing flushed file at 1732742153004Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7db94e56: reopening flushed file at 1732742153412 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65a23709: reopening flushed file at 1732742153415 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@650af797: reopening flushed file at 1732742153419 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 49ff3edd8f78b405b2b26bdd2aa6e347 in 1651ms, sequenceid=485, compaction requested=true at 1732742153423 (+4 ms)Writing region close event to WAL at 1732742153424 (+1 ms)Running coprocessor post-close hooks at 1732742153427 (+3 ms)Closed at 1732742153427 2024-11-27T21:15:53,429 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(157): Closed 49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,430 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=49ff3edd8f78b405b2b26bdd2aa6e347, regionState=CLOSED 2024-11-27T21:15:53,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure 49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:15:53,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-27T21:15:53,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseRegionProcedure 49ff3edd8f78b405b2b26bdd2aa6e347, server=4585c39bed03,37381,1732742022265 in 1.8120 sec 2024-11-27T21:15:53,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-27T21:15:53,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49ff3edd8f78b405b2b26bdd2aa6e347, UNASSIGN in 1.8160 sec 2024-11-27T21:15:53,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-27T21:15:53,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8190 sec 2024-11-27T21:15:53,438 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742153438"}]},"ts":"1732742153438"} 2024-11-27T21:15:53,440 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T21:15:53,440 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T21:15:53,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8280 sec 2024-11-27T21:15:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T21:15:53,760 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:53,760 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T21:15:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:53,762 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-27T21:15:53,762 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=121, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:53,765 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,767 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/recovered.edits] 2024-11-27T21:15:53,769 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6ea7dd5754be43f8b8ac735484f48839 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/6ea7dd5754be43f8b8ac735484f48839 2024-11-27T21:15:53,770 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9f82bee97c544e28b305e7a47cc87081 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/9f82bee97c544e28b305e7a47cc87081 2024-11-27T21:15:53,771 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/a12eee94550247b5895db3267abc5256 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/a12eee94550247b5895db3267abc5256 2024-11-27T21:15:53,772 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/abeb51be13a64e8295531df3fc6d503e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/A/abeb51be13a64e8295531df3fc6d503e 2024-11-27T21:15:53,773 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/8bd38044e9f8442dbbdaebbe38f8aff6 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/8bd38044e9f8442dbbdaebbe38f8aff6 2024-11-27T21:15:53,774 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/c20e3ebbe50a4cfe89228b8d30a4a9be to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/c20e3ebbe50a4cfe89228b8d30a4a9be 2024-11-27T21:15:53,775 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/dce287c8b6b146d5a0d979cc2049b20b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/dce287c8b6b146d5a0d979cc2049b20b 2024-11-27T21:15:53,776 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/ed5ce6c928564b7b950434a7e27f78e3 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/B/ed5ce6c928564b7b950434a7e27f78e3 2024-11-27T21:15:53,777 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/6cff3376ce7142eb8c3b59bc37c6b154 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/6cff3376ce7142eb8c3b59bc37c6b154 2024-11-27T21:15:53,778 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/72a05f9c25e14aa7a2e06f7f990a178f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/72a05f9c25e14aa7a2e06f7f990a178f 2024-11-27T21:15:53,779 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/741c3000aa9a4f3aa7ae8366d4ec290e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/741c3000aa9a4f3aa7ae8366d4ec290e 2024-11-27T21:15:53,780 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e0110edcdcbd4b26a6538630cc67b4f2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/C/e0110edcdcbd4b26a6538630cc67b4f2 2024-11-27T21:15:53,782 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/recovered.edits/488.seqid to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347/recovered.edits/488.seqid 2024-11-27T21:15:53,782 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,782 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T21:15:53,783 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T21:15:53,783 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T21:15:53,785 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270f598fea251e4d2ea0ee7ae33c7048cc_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270f598fea251e4d2ea0ee7ae33c7048cc_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,786 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271424c4eb3199451dbb90337724913bd4_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271424c4eb3199451dbb90337724913bd4_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,787 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272053eaf28e9f4d0a85c0afccf7b14302_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272053eaf28e9f4d0a85c0afccf7b14302_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,788 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112747d7928742fa447bb6219610a2d618b0_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112747d7928742fa447bb6219610a2d618b0_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,788 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274d4237aa03634a75bf94f888b2547fe7_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274d4237aa03634a75bf94f888b2547fe7_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,789 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127564da156e8684393b1f39c562e56d61b_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127564da156e8684393b1f39c562e56d61b_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,790 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112771ae73bdf5cf4bf29c6f1edd14bdac19_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112771ae73bdf5cf4bf29c6f1edd14bdac19_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,791 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277bc3ed3a1bb24e12aacfb99cae568614_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277bc3ed3a1bb24e12aacfb99cae568614_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,791 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277fc504e398b84a35b104486954eb0979_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277fc504e398b84a35b104486954eb0979_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,792 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127876cca9c945f4d9db8514c7e7581642e_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127876cca9c945f4d9db8514c7e7581642e_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,793 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112796b9bef34b3b4773bd8aea87ab97b5b3_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112796b9bef34b3b4773bd8aea87ab97b5b3_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,794 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b41c55f581cd4b4092a93b4c21b76f0c_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b41c55f581cd4b4092a93b4c21b76f0c_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,795 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b9d68b7ae0664d6dbbd2216acb814946_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b9d68b7ae0664d6dbbd2216acb814946_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,795 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bc6e35b4f84c4bd780b77b5530381441_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bc6e35b4f84c4bd780b77b5530381441_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,796 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cf5395e67388401e91520cd3a9450c45_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cf5395e67388401e91520cd3a9450c45_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,797 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127dcd12eb06ce34df7a589f993b4900d4c_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127dcd12eb06ce34df7a589f993b4900d4c_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,798 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127eaf04c345f3e46f0a79164e1a4314e08_49ff3edd8f78b405b2b26bdd2aa6e347 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127eaf04c345f3e46f0a79164e1a4314e08_49ff3edd8f78b405b2b26bdd2aa6e347 2024-11-27T21:15:53,798 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T21:15:53,800 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=121, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:53,801 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T21:15:53,803 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T21:15:53,804 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=121, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:53,804 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T21:15:53,804 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732742153804"}]},"ts":"9223372036854775807"} 2024-11-27T21:15:53,805 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T21:15:53,805 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 49ff3edd8f78b405b2b26bdd2aa6e347, NAME => 'TestAcidGuarantees,,1732742118529.49ff3edd8f78b405b2b26bdd2aa6e347.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T21:15:53,805 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T21:15:53,805 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732742153805"}]},"ts":"9223372036854775807"} 2024-11-27T21:15:53,807 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T21:15:53,807 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=121, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:53,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-27T21:15:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-27T21:15:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T21:15:53,870 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:53,879 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 250), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=328 (was 424), ProcessCount=11 (was 11), AvailableMemoryMB=7975 (was 8519) 2024-11-27T21:15:53,887 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=328, ProcessCount=11, AvailableMemoryMB=7975 2024-11-27T21:15:53,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:15:53,889 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:15:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T21:15:53,890 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T21:15:53,890 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 122 2024-11-27T21:15:53,890 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-27T21:15:53,891 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T21:15:53,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742282_1458 (size=1061) 2024-11-27T21:15:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-27T21:15:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-27T21:15:54,297 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de 2024-11-27T21:15:54,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742283_1459 (size=53) 2024-11-27T21:15:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-27T21:15:54,702 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:15:54,702 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cf3e016fc40bf5fab56083bda0e72b03, disabling compactions & flushes 2024-11-27T21:15:54,702 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:54,702 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:54,702 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. after waiting 0 ms 2024-11-27T21:15:54,702 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:54,702 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:54,702 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cf3e016fc40bf5fab56083bda0e72b03: Waiting for close lock at 1732742154702Disabling compacts and flushes for region at 1732742154702Disabling writes for close at 1732742154702Writing region close event to WAL at 1732742154702Closed at 1732742154702 2024-11-27T21:15:54,703 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T21:15:54,704 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732742154703"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732742154703"}]},"ts":"1732742154703"} 2024-11-27T21:15:54,705 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T21:15:54,706 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T21:15:54,706 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742154706"}]},"ts":"1732742154706"} 2024-11-27T21:15:54,707 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T21:15:54,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf3e016fc40bf5fab56083bda0e72b03, ASSIGN}] 2024-11-27T21:15:54,708 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf3e016fc40bf5fab56083bda0e72b03, ASSIGN 2024-11-27T21:15:54,708 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf3e016fc40bf5fab56083bda0e72b03, ASSIGN; state=OFFLINE, location=4585c39bed03,37381,1732742022265; forceNewPlan=false, retain=false 2024-11-27T21:15:54,859 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=cf3e016fc40bf5fab56083bda0e72b03, regionState=OPENING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:15:54,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf3e016fc40bf5fab56083bda0e72b03, ASSIGN because future has completed 2024-11-27T21:15:54,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:15:55,015 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:55,015 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7752): Opening region: {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:15:55,015 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:55,015 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:15:55,016 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7794): checking encryption for cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:55,016 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7797): checking classloading for cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:55,017 INFO [StoreOpener-cf3e016fc40bf5fab56083bda0e72b03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:55,017 INFO [StoreOpener-cf3e016fc40bf5fab56083bda0e72b03-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:15:55,018 INFO [StoreOpener-cf3e016fc40bf5fab56083bda0e72b03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf3e016fc40bf5fab56083bda0e72b03 columnFamilyName A 2024-11-27T21:15:55,018 DEBUG [StoreOpener-cf3e016fc40bf5fab56083bda0e72b03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:55,018 INFO [StoreOpener-cf3e016fc40bf5fab56083bda0e72b03-1 {}] regionserver.HStore(327): Store=cf3e016fc40bf5fab56083bda0e72b03/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:15:55,018 INFO [StoreOpener-cf3e016fc40bf5fab56083bda0e72b03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:55,019 INFO [StoreOpener-cf3e016fc40bf5fab56083bda0e72b03-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:15:55,019 INFO [StoreOpener-cf3e016fc40bf5fab56083bda0e72b03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf3e016fc40bf5fab56083bda0e72b03 columnFamilyName B 2024-11-27T21:15:55,019 DEBUG [StoreOpener-cf3e016fc40bf5fab56083bda0e72b03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:55,019 INFO [StoreOpener-cf3e016fc40bf5fab56083bda0e72b03-1 {}] regionserver.HStore(327): Store=cf3e016fc40bf5fab56083bda0e72b03/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:15:55,020 INFO [StoreOpener-cf3e016fc40bf5fab56083bda0e72b03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:55,020 INFO [StoreOpener-cf3e016fc40bf5fab56083bda0e72b03-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:15:55,020 INFO [StoreOpener-cf3e016fc40bf5fab56083bda0e72b03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf3e016fc40bf5fab56083bda0e72b03 columnFamilyName C 2024-11-27T21:15:55,020 DEBUG [StoreOpener-cf3e016fc40bf5fab56083bda0e72b03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:15:55,021 INFO [StoreOpener-cf3e016fc40bf5fab56083bda0e72b03-1 {}] regionserver.HStore(327): Store=cf3e016fc40bf5fab56083bda0e72b03/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:15:55,021 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:55,021 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1038): replaying wal for cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:55,021 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:55,022 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:55,022 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1048): stopping wal replay for cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:55,022 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1060): Cleaning up temporary data for cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:55,023 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:15:55,023 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1093): writing seq id for cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:55,025 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T21:15:55,025 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1114): Opened cf3e016fc40bf5fab56083bda0e72b03; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60985519, jitterRate=-0.09124495089054108}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:15:55,025 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:55,025 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1006): Region open journal for cf3e016fc40bf5fab56083bda0e72b03: Running coprocessor pre-open hook at 1732742155016Writing region info on filesystem at 1732742155016Initializing all the Stores at 1732742155016Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742155016Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742155016Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742155016Cleaning up temporary data from old regions at 1732742155022 (+6 ms)Running coprocessor post-open hooks at 1732742155025 (+3 ms)Region opened successfully at 1732742155025 2024-11-27T21:15:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-27T21:15:55,030 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., pid=124, masterSystemTime=1732742155012 2024-11-27T21:15:55,032 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:55,032 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:55,032 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=cf3e016fc40bf5fab56083bda0e72b03, regionState=OPEN, openSeqNum=2, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:15:55,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:15:55,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-27T21:15:55,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; OpenRegionProcedure cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 in 173 msec 2024-11-27T21:15:55,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-27T21:15:55,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf3e016fc40bf5fab56083bda0e72b03, ASSIGN in 329 msec 2024-11-27T21:15:55,038 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T21:15:55,038 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742155038"}]},"ts":"1732742155038"} 2024-11-27T21:15:55,039 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T21:15:55,040 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T21:15:55,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-27T21:15:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-27T21:15:56,039 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:56,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,040 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:56,040 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:56,041 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a5bf3d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:56,043 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:56,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75cd6d32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,044 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:56,045 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:56,046 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T21:15:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 4585c39bed03,38503,1732742021440 2024-11-27T21:15:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2d32caf0 2024-11-27T21:15:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T21:15:56,047 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T21:15:56,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,048 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:56,048 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:56,049 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3782f0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:56,050 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:56,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6948312a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,051 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:56,051 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:56,051 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:56,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2573d966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:56,052 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5628a4df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,053 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:56,054 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:56,054 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27a0ef39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:56,055 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a4701e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,056 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,056 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:56,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:56,057 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24ad392d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:56,058 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:56,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ca536f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,059 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:56,059 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:56,059 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:56,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:56,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d82098e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:56,060 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10b78d78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:56,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:56,062 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@599dca55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:56,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:56,063 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:56,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7136b4dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,064 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:56,064 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:56,065 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16e9a2a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:56,066 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f7e187c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,067 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:56,067 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:56,067 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d1f697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:56,069 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:56,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e6f6dfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:56,070 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:56,070 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b94eaa5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:56,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:56,071 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ff2961, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:15:56,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:15:56,073 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:15:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:15:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:15:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ed8fbd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:15:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:15:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:15:56,074 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:15:56,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130e9895, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:15:56,075 DEBUG [Thread-2072 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:56,075 DEBUG [Thread-2059 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:56,075 DEBUG [Thread-2057 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:56,075 DEBUG [Thread-2068 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:56,075 DEBUG [Thread-2061 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:56,075 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:15:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-27T21:15:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:56,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:56,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T21:15:56,077 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:15:56,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:56,078 DEBUG [Thread-2076 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:56,078 DEBUG [Thread-2074 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:56,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:56,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:56,078 DEBUG [Thread-2063 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:56,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:56,079 DEBUG [Thread-2076 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:56,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:56,079 DEBUG [Thread-2076 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:56,079 DEBUG [Thread-2063 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:56,079 DEBUG [Thread-2063 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:56,079 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:15:56,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:15:56,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:56,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:15:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:15:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:15:56,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:15:56,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:56,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:15:56,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:15:56,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:15:56,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:56,082 DEBUG [Thread-2065 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:56,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:56,084 DEBUG [Thread-2070 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:15:56,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:56,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:56,085 DEBUG [Thread-2070 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:15:56,085 DEBUG [Thread-2070 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:15:56,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:56,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:15:56,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:15:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:15:56,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2] 2024-11-27T21:15:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:56,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:15:56,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:15:56,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:56,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:15:56,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:56,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:15:56,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:56,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/9836a419f0754acebebb22b99f06a1bc is 50, key is test_row_0/A:col10/1732742156086/Put/seqid=0 2024-11-27T21:15:56,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742284_1460 (size=12001) 2024-11-27T21:15:56,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742216133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742216134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742216135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742156100, max timestamp=1732742156146Num uniques 859; 2024-11-27T21:15:56,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:15:56,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742156100, max timestamp=1732742156146Num uniques 859; 2024-11-27T21:15:56,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742156100, max timestamp=1732742156146Num uniques 859; 2024-11-27T21:15:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:15:56,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:15:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T21:15:56,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T21:15:56,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:56,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:15:56,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:56,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742216220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742216221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742216258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742216258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742216259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742216338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742216348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T21:15:56,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:56,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:15:56,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:56,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T21:15:56,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742216479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742216479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742216479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/9836a419f0754acebebb22b99f06a1bc 2024-11-27T21:15:56,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T21:15:56,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:56,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:15:56,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:56,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/d33cf303383445ed9341b6bc7f740f35 is 50, key is test_row_0/B:col10/1732742156086/Put/seqid=0 2024-11-27T21:15:56,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742285_1461 (size=12001) 2024-11-27T21:15:56,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742216559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742216569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T21:15:56,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:56,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:15:56,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:56,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T21:15:56,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742216789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742216799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742216799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T21:15:56,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:56,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:15:56,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:56,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742216879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742216879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:56,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/d33cf303383445ed9341b6bc7f740f35 2024-11-27T21:15:56,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/4d982433417f408ba3e5ef34a22520c5 is 50, key is test_row_0/C:col10/1732742156086/Put/seqid=0 2024-11-27T21:15:56,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742286_1462 (size=12001) 2024-11-27T21:15:56,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/4d982433417f408ba3e5ef34a22520c5 2024-11-27T21:15:56,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/9836a419f0754acebebb22b99f06a1bc as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9836a419f0754acebebb22b99f06a1bc 2024-11-27T21:15:56,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9836a419f0754acebebb22b99f06a1bc, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T21:15:56,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/d33cf303383445ed9341b6bc7f740f35 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d33cf303383445ed9341b6bc7f740f35 2024-11-27T21:15:56,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d33cf303383445ed9341b6bc7f740f35, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T21:15:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/4d982433417f408ba3e5ef34a22520c5 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/4d982433417f408ba3e5ef34a22520c5 2024-11-27T21:15:56,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T21:15:56,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:56,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:15:56,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:56,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:15:57,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/4d982433417f408ba3e5ef34a22520c5, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T21:15:57,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cf3e016fc40bf5fab56083bda0e72b03 in 907ms, sequenceid=12, compaction requested=false 2024-11-27T21:15:57,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:15:57,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T21:15:57,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:57,151 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-27T21:15:57,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:15:57,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:57,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:15:57,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:57,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:15:57,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:57,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/5fdb9746d79643a884c5b1e9148d01c4 is 50, key is test_row_0/A:col10/1732742156215/Put/seqid=0 2024-11-27T21:15:57,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742287_1463 (size=12001) 2024-11-27T21:15:57,162 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/5fdb9746d79643a884c5b1e9148d01c4 2024-11-27T21:15:57,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/21bc047e98dc493e8c57f7d75fb5bfe0 is 50, key is test_row_0/B:col10/1732742156215/Put/seqid=0 2024-11-27T21:15:57,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742288_1464 (size=12001) 2024-11-27T21:15:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T21:15:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:57,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:15:57,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742217331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:57,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:57,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742217335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742217341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:57,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742217399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:57,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742217399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:57,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742217449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:57,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742217458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:57,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742217459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:57,578 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/21bc047e98dc493e8c57f7d75fb5bfe0 2024-11-27T21:15:57,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/07da68eac79b448e9e2d1b7ba4970bd4 is 50, key is test_row_0/C:col10/1732742156215/Put/seqid=0 2024-11-27T21:15:57,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742289_1465 (size=12001) 2024-11-27T21:15:57,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742217669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:57,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742217678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:57,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:57,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742217679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:57,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:57,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:57,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742217989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:57,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742217989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:57,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742217989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:57,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:57,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:57,996 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/07da68eac79b448e9e2d1b7ba4970bd4 2024-11-27T21:15:58,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/5fdb9746d79643a884c5b1e9148d01c4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/5fdb9746d79643a884c5b1e9148d01c4 2024-11-27T21:15:58,003 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/5fdb9746d79643a884c5b1e9148d01c4, entries=150, sequenceid=48, filesize=11.7 K 2024-11-27T21:15:58,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/21bc047e98dc493e8c57f7d75fb5bfe0 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/21bc047e98dc493e8c57f7d75fb5bfe0 2024-11-27T21:15:58,007 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/21bc047e98dc493e8c57f7d75fb5bfe0, entries=150, sequenceid=48, filesize=11.7 K 2024-11-27T21:15:58,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/07da68eac79b448e9e2d1b7ba4970bd4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/07da68eac79b448e9e2d1b7ba4970bd4 2024-11-27T21:15:58,011 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/07da68eac79b448e9e2d1b7ba4970bd4, entries=150, sequenceid=48, filesize=11.7 K 2024-11-27T21:15:58,012 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for cf3e016fc40bf5fab56083bda0e72b03 in 861ms, sequenceid=48, compaction requested=false 2024-11-27T21:15:58,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:15:58,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-27T21:15:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-27T21:15:58,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-27T21:15:58,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9350 sec 2024-11-27T21:15:58,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.9410 sec 2024-11-27T21:15:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T21:15:58,230 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:15:58,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:15:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-27T21:15:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T21:15:58,232 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:15:58,232 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:15:58,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:15:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T21:15:58,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T21:15:58,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:58,385 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-27T21:15:58,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:15:58,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:58,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:15:58,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:58,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:15:58,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:15:58,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/0423ade818da4534ac2fd089c0619bab is 50, key is test_row_0/A:col10/1732742157340/Put/seqid=0 2024-11-27T21:15:58,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742290_1466 (size=12001) 2024-11-27T21:15:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:58,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:15:58,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742158467, max timestamp=1732742158529Num uniques 859; 2024-11-27T21:15:58,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:15:58,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742158467, max timestamp=1732742158529Num uniques 859; 2024-11-27T21:15:58,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:15:58,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742158467, max timestamp=1732742158529Num uniques 859; 2024-11-27T21:15:58,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:15:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T21:15:58,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742218567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:58,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742218568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:58,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:58,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742218570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:58,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:58,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:58,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742218571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:58,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:58,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:58,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742218572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:58,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742218679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:58,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:58,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742218688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:58,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742218689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:58,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742218689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:58,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742218699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:58,794 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/0423ade818da4534ac2fd089c0619bab 2024-11-27T21:15:58,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/6f66d12973f44a95a9184af7f3adea48 is 50, key is test_row_0/B:col10/1732742157340/Put/seqid=0 2024-11-27T21:15:58,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742291_1467 (size=12001) 2024-11-27T21:15:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T21:15:58,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742218899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:58,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742218909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:58,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:58,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742218909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:58,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:58,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742218909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:58,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:58,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:58,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742218918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:58,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:58,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:58,946 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T21:15:59,205 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/6f66d12973f44a95a9184af7f3adea48 2024-11-27T21:15:59,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/3d383d1d29054eec87e36288ff4e7a82 is 50, key is test_row_0/C:col10/1732742157340/Put/seqid=0 2024-11-27T21:15:59,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742292_1468 (size=12001) 2024-11-27T21:15:59,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742219219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:59,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742219229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:59,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:59,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742219229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742219229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:59,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742219238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T21:15:59,624 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/3d383d1d29054eec87e36288ff4e7a82 2024-11-27T21:15:59,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/0423ade818da4534ac2fd089c0619bab as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/0423ade818da4534ac2fd089c0619bab 2024-11-27T21:15:59,630 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/0423ade818da4534ac2fd089c0619bab, entries=150, sequenceid=59, filesize=11.7 K 2024-11-27T21:15:59,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/6f66d12973f44a95a9184af7f3adea48 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6f66d12973f44a95a9184af7f3adea48 2024-11-27T21:15:59,634 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6f66d12973f44a95a9184af7f3adea48, entries=150, sequenceid=59, filesize=11.7 K 2024-11-27T21:15:59,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/3d383d1d29054eec87e36288ff4e7a82 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/3d383d1d29054eec87e36288ff4e7a82 2024-11-27T21:15:59,638 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/3d383d1d29054eec87e36288ff4e7a82, entries=150, sequenceid=59, filesize=11.7 K 2024-11-27T21:15:59,639 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cf3e016fc40bf5fab56083bda0e72b03 in 1254ms, sequenceid=59, compaction requested=true 2024-11-27T21:15:59,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:15:59,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:15:59,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-27T21:15:59,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-27T21:15:59,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-27T21:15:59,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4070 sec 2024-11-27T21:15:59,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.4130 sec 2024-11-27T21:15:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:15:59,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-27T21:15:59,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:15:59,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:59,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:15:59,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:59,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:15:59,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:15:59,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/6654853c912e4ae99cd5c0141e346587 is 50, key is test_row_0/A:col10/1732742158568/Put/seqid=0 2024-11-27T21:15:59,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742293_1469 (size=14341) 2024-11-27T21:15:59,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742219763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:59,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742219766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:59,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742219768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742219769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:59,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742219774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:59,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742219879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:59,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:59,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742219889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:59,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742219889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:59,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742219889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:15:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:15:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742219889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:15:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:15:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742220099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742220109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742220109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742220110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742220110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/6654853c912e4ae99cd5c0141e346587 2024-11-27T21:16:00,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/0f8f74fe23e848b2938a8e593cb2bc69 is 50, key is test_row_0/B:col10/1732742158568/Put/seqid=0 2024-11-27T21:16:00,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742294_1470 (size=12001) 2024-11-27T21:16:00,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/0f8f74fe23e848b2938a8e593cb2bc69 2024-11-27T21:16:00,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/ca8a935beca746169ef7a48bcc11c60e is 50, key is test_row_0/C:col10/1732742158568/Put/seqid=0 2024-11-27T21:16:00,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742295_1471 (size=12001) 2024-11-27T21:16:00,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/ca8a935beca746169ef7a48bcc11c60e 2024-11-27T21:16:00,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/6654853c912e4ae99cd5c0141e346587 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/6654853c912e4ae99cd5c0141e346587 2024-11-27T21:16:00,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/6654853c912e4ae99cd5c0141e346587, entries=200, sequenceid=94, filesize=14.0 K 2024-11-27T21:16:00,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/0f8f74fe23e848b2938a8e593cb2bc69 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/0f8f74fe23e848b2938a8e593cb2bc69 2024-11-27T21:16:00,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/0f8f74fe23e848b2938a8e593cb2bc69, entries=150, sequenceid=94, filesize=11.7 K 2024-11-27T21:16:00,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/ca8a935beca746169ef7a48bcc11c60e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/ca8a935beca746169ef7a48bcc11c60e 2024-11-27T21:16:00,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/ca8a935beca746169ef7a48bcc11c60e, entries=150, sequenceid=94, filesize=11.7 K 2024-11-27T21:16:00,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for cf3e016fc40bf5fab56083bda0e72b03 in 457ms, sequenceid=94, compaction requested=true 2024-11-27T21:16:00,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:00,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:16:00,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:00,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:16:00,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:00,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:16:00,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:00,205 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:00,205 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:00,206 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:00,206 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/A is initiating minor compaction (all files) 2024-11-27T21:16:00,206 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/A in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:00,206 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9836a419f0754acebebb22b99f06a1bc, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/5fdb9746d79643a884c5b1e9148d01c4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/0423ade818da4534ac2fd089c0619bab, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/6654853c912e4ae99cd5c0141e346587] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=49.2 K 2024-11-27T21:16:00,206 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:00,206 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/B is initiating minor compaction (all files) 2024-11-27T21:16:00,206 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/B in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:00,206 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d33cf303383445ed9341b6bc7f740f35, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/21bc047e98dc493e8c57f7d75fb5bfe0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6f66d12973f44a95a9184af7f3adea48, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/0f8f74fe23e848b2938a8e593cb2bc69] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=46.9 K 2024-11-27T21:16:00,207 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9836a419f0754acebebb22b99f06a1bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732742156086 2024-11-27T21:16:00,207 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting d33cf303383445ed9341b6bc7f740f35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732742156086 2024-11-27T21:16:00,207 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fdb9746d79643a884c5b1e9148d01c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732742156215 2024-11-27T21:16:00,207 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 21bc047e98dc493e8c57f7d75fb5bfe0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732742156215 2024-11-27T21:16:00,207 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0423ade818da4534ac2fd089c0619bab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732742157315 2024-11-27T21:16:00,207 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f66d12973f44a95a9184af7f3adea48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732742157315 2024-11-27T21:16:00,207 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6654853c912e4ae99cd5c0141e346587, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732742158553 2024-11-27T21:16:00,208 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f8f74fe23e848b2938a8e593cb2bc69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732742158568 2024-11-27T21:16:00,216 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#B#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:00,216 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#A#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:00,217 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/a90081ab6a6047228cae4567708e58ea is 50, key is test_row_0/B:col10/1732742158568/Put/seqid=0 2024-11-27T21:16:00,217 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/a880a5b6ac6f4a76a555b281ee62a5e6 is 50, key is test_row_0/A:col10/1732742158568/Put/seqid=0 2024-11-27T21:16:00,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742296_1472 (size=12139) 2024-11-27T21:16:00,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742297_1473 (size=12139) 2024-11-27T21:16:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T21:16:00,379 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:00,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:16:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-27T21:16:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T21:16:00,382 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:16:00,382 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:16:00,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:16:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:00,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-27T21:16:00,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:00,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:00,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:00,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:00,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:00,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:00,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/5127283fcac34be99fff394ec15142f2 is 50, key is test_row_0/A:col10/1732742160423/Put/seqid=0 2024-11-27T21:16:00,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742298_1474 (size=14341) 2024-11-27T21:16:00,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742160465, max timestamp=1732742160473Num uniques 859; 2024-11-27T21:16:00,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:16:00,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742160465, max timestamp=1732742160473Num uniques 859; 2024-11-27T21:16:00,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:16:00,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742160465, max timestamp=1732742160473Num uniques 859; 2024-11-27T21:16:00,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:16:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T21:16:00,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742220510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742220511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742220512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742220513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:16:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:00,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:00,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742220520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:00,627 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/a90081ab6a6047228cae4567708e58ea as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/a90081ab6a6047228cae4567708e58ea 2024-11-27T21:16:00,627 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/a880a5b6ac6f4a76a555b281ee62a5e6 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/a880a5b6ac6f4a76a555b281ee62a5e6 2024-11-27T21:16:00,631 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/A of cf3e016fc40bf5fab56083bda0e72b03 into a880a5b6ac6f4a76a555b281ee62a5e6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:00,631 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/B of cf3e016fc40bf5fab56083bda0e72b03 into a90081ab6a6047228cae4567708e58ea(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:00,631 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:00,631 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:00,631 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/B, priority=12, startTime=1732742160205; duration=0sec 2024-11-27T21:16:00,631 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/A, priority=12, startTime=1732742160204; duration=0sec 2024-11-27T21:16:00,631 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:00,632 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:B 2024-11-27T21:16:00,632 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:00,632 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:A 2024-11-27T21:16:00,632 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:00,632 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:00,633 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/C is initiating minor compaction (all files) 2024-11-27T21:16:00,633 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/C in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:00,633 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/4d982433417f408ba3e5ef34a22520c5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/07da68eac79b448e9e2d1b7ba4970bd4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/3d383d1d29054eec87e36288ff4e7a82, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/ca8a935beca746169ef7a48bcc11c60e] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=46.9 K 2024-11-27T21:16:00,633 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d982433417f408ba3e5ef34a22520c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732742156086 2024-11-27T21:16:00,633 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 07da68eac79b448e9e2d1b7ba4970bd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732742156215 2024-11-27T21:16:00,634 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d383d1d29054eec87e36288ff4e7a82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732742157315 2024-11-27T21:16:00,634 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting ca8a935beca746169ef7a48bcc11c60e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732742158568 2024-11-27T21:16:00,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742220628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742220639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742220639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742220639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,643 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#C#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:00,644 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/a6f2587577194e24abd8bc9ddf38fd98 is 50, key is test_row_0/C:col10/1732742158568/Put/seqid=0 2024-11-27T21:16:00,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742299_1475 (size=12139) 2024-11-27T21:16:00,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742220640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:16:00,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:00,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:00,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:00,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:00,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T21:16:00,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/5127283fcac34be99fff394ec15142f2 2024-11-27T21:16:00,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:16:00,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:00,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:00,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:00,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:00,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:00,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/8caa4e83078d459c882bce3c95abc8c7 is 50, key is test_row_0/B:col10/1732742160423/Put/seqid=0 2024-11-27T21:16:00,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742300_1476 (size=12001) 2024-11-27T21:16:00,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742220849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742220849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742220849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742220849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742220859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:00,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:00,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:00,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:16:00,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:00,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:00,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:00,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:00,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T21:16:01,051 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/a6f2587577194e24abd8bc9ddf38fd98 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a6f2587577194e24abd8bc9ddf38fd98 2024-11-27T21:16:01,055 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/C of cf3e016fc40bf5fab56083bda0e72b03 into a6f2587577194e24abd8bc9ddf38fd98(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:01,055 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:01,055 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/C, priority=12, startTime=1732742160205; duration=0sec 2024-11-27T21:16:01,055 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:01,055 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:C 2024-11-27T21:16:01,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:16:01,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:01,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:01,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:01,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742221169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:01,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742221169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:01,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742221169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742221169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:01,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:01,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742221179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:01,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/8caa4e83078d459c882bce3c95abc8c7 2024-11-27T21:16:01,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/8949b9b71e824fbe92ded10c9d3fc3e3 is 50, key is test_row_0/C:col10/1732742160423/Put/seqid=0 2024-11-27T21:16:01,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742301_1477 (size=12001) 2024-11-27T21:16:01,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:16:01,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:01,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:01,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:01,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:16:01,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:01,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:01,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:01,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T21:16:01,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:16:01,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:01,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:01,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:01,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/8949b9b71e824fbe92ded10c9d3fc3e3 2024-11-27T21:16:01,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/5127283fcac34be99fff394ec15142f2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/5127283fcac34be99fff394ec15142f2 2024-11-27T21:16:01,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/5127283fcac34be99fff394ec15142f2, entries=200, sequenceid=105, filesize=14.0 K 2024-11-27T21:16:01,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/8caa4e83078d459c882bce3c95abc8c7 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8caa4e83078d459c882bce3c95abc8c7 2024-11-27T21:16:01,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8caa4e83078d459c882bce3c95abc8c7, entries=150, sequenceid=105, filesize=11.7 K 2024-11-27T21:16:01,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/8949b9b71e824fbe92ded10c9d3fc3e3 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/8949b9b71e824fbe92ded10c9d3fc3e3 2024-11-27T21:16:01,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/8949b9b71e824fbe92ded10c9d3fc3e3, entries=150, sequenceid=105, filesize=11.7 K 2024-11-27T21:16:01,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cf3e016fc40bf5fab56083bda0e72b03 in 1260ms, sequenceid=105, compaction requested=false 2024-11-27T21:16:01,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:01,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-27T21:16:01,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:01,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:01,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:01,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:01,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:01,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:01,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/f24f0c9585ab41bc9d981f0d50b9b27f is 50, key is test_row_0/A:col10/1732742160509/Put/seqid=0 2024-11-27T21:16:01,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742302_1478 (size=14541) 2024-11-27T21:16:01,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/f24f0c9585ab41bc9d981f0d50b9b27f 2024-11-27T21:16:01,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/86285e2662824bd1aa33a14054a798c6 is 50, key is test_row_0/B:col10/1732742160509/Put/seqid=0 2024-11-27T21:16:01,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742221699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:01,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742303_1479 (size=12151) 2024-11-27T21:16:01,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:01,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742221704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:01,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742221712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:01,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742221714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:01,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742221714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:01,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:01,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:01,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:16:01,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:01,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:01,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:01,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742221819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742221819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:01,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742221839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:01,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:01,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742221839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:01,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:01,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:01,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:01,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:16:01,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:01,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:01,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:01,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:02,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742222039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742222039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:02,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742222049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742222049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:02,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:16:02,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:02,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:02,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:02,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:02,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:02,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/86285e2662824bd1aa33a14054a798c6 2024-11-27T21:16:02,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/9e122496683048e38e8dddb6aeb4a061 is 50, key is test_row_0/C:col10/1732742160509/Put/seqid=0 2024-11-27T21:16:02,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742304_1480 (size=12151) 2024-11-27T21:16:02,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/9e122496683048e38e8dddb6aeb4a061 2024-11-27T21:16:02,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/f24f0c9585ab41bc9d981f0d50b9b27f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/f24f0c9585ab41bc9d981f0d50b9b27f 2024-11-27T21:16:02,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/f24f0c9585ab41bc9d981f0d50b9b27f, entries=200, sequenceid=144, filesize=14.2 K 2024-11-27T21:16:02,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/86285e2662824bd1aa33a14054a798c6 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/86285e2662824bd1aa33a14054a798c6 2024-11-27T21:16:02,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/86285e2662824bd1aa33a14054a798c6, entries=150, sequenceid=144, filesize=11.9 K 2024-11-27T21:16:02,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/9e122496683048e38e8dddb6aeb4a061 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9e122496683048e38e8dddb6aeb4a061 2024-11-27T21:16:02,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9e122496683048e38e8dddb6aeb4a061, entries=150, sequenceid=144, filesize=11.9 K 2024-11-27T21:16:02,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for cf3e016fc40bf5fab56083bda0e72b03 in 445ms, sequenceid=144, compaction requested=true 2024-11-27T21:16:02,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:02,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:16:02,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:02,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:16:02,140 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:02,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:02,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:16:02,140 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:02,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:02,141 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:02,141 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/B is initiating minor compaction (all files) 2024-11-27T21:16:02,141 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/B in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:02,141 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/a90081ab6a6047228cae4567708e58ea, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8caa4e83078d459c882bce3c95abc8c7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/86285e2662824bd1aa33a14054a798c6] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=35.4 K 2024-11-27T21:16:02,142 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41021 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:02,142 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/A is initiating minor compaction (all files) 2024-11-27T21:16:02,142 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/A in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:02,142 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting a90081ab6a6047228cae4567708e58ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732742158568 2024-11-27T21:16:02,142 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/a880a5b6ac6f4a76a555b281ee62a5e6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/5127283fcac34be99fff394ec15142f2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/f24f0c9585ab41bc9d981f0d50b9b27f] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=40.1 K 2024-11-27T21:16:02,143 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 8caa4e83078d459c882bce3c95abc8c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732742159761 2024-11-27T21:16:02,143 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting a880a5b6ac6f4a76a555b281ee62a5e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732742158568 2024-11-27T21:16:02,143 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 86285e2662824bd1aa33a14054a798c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732742160509 2024-11-27T21:16:02,143 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5127283fcac34be99fff394ec15142f2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732742159761 2024-11-27T21:16:02,143 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting f24f0c9585ab41bc9d981f0d50b9b27f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732742160501 2024-11-27T21:16:02,152 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#A#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:02,153 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/fe0903bf936f4db9b1f2678028ae42f0 is 50, key is test_row_0/A:col10/1732742160509/Put/seqid=0 2024-11-27T21:16:02,155 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#B#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:02,156 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/5ab27b9e698d462a90c5a8ccb0622905 is 50, key is test_row_0/B:col10/1732742160509/Put/seqid=0 2024-11-27T21:16:02,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742306_1482 (size=12391) 2024-11-27T21:16:02,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742305_1481 (size=12391) 2024-11-27T21:16:02,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T21:16:02,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:02,218 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:16:02,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:02,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:02,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:02,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:02,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:02,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:02,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/c18d76137a974c2fb05a0638fac31d8b is 50, key is test_row_1/A:col10/1732742161712/Put/seqid=0 2024-11-27T21:16:02,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742307_1483 (size=9757) 2024-11-27T21:16:02,227 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/c18d76137a974c2fb05a0638fac31d8b 2024-11-27T21:16:02,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/1579f6ca0e2645b49a76f7dd6583ec2d is 50, key is test_row_1/B:col10/1732742161712/Put/seqid=0 2024-11-27T21:16:02,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742308_1484 (size=9757) 2024-11-27T21:16:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:02,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:02,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742162377, max timestamp=1732742162443Num uniques 859; 2024-11-27T21:16:02,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:02,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742162377, max timestamp=1732742162443Num uniques 859; 2024-11-27T21:16:02,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:02,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742162377, max timestamp=1732742162443Num uniques 859; 2024-11-27T21:16:02,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:02,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742222483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:02,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742222490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:02,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742222491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742222493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:02,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T21:16:02,573 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/5ab27b9e698d462a90c5a8ccb0622905 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/5ab27b9e698d462a90c5a8ccb0622905 2024-11-27T21:16:02,574 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/fe0903bf936f4db9b1f2678028ae42f0 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/fe0903bf936f4db9b1f2678028ae42f0 2024-11-27T21:16:02,585 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/A of cf3e016fc40bf5fab56083bda0e72b03 into fe0903bf936f4db9b1f2678028ae42f0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:02,585 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:02,585 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/A, priority=13, startTime=1732742162140; duration=0sec 2024-11-27T21:16:02,585 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:02,585 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:A 2024-11-27T21:16:02,585 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:02,587 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/B of cf3e016fc40bf5fab56083bda0e72b03 into 5ab27b9e698d462a90c5a8ccb0622905(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:02,587 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:02,587 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/B, priority=13, startTime=1732742162140; duration=0sec 2024-11-27T21:16:02,587 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:02,587 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:B 2024-11-27T21:16:02,588 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:02,588 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/C is initiating minor compaction (all files) 2024-11-27T21:16:02,588 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/C in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:02,588 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a6f2587577194e24abd8bc9ddf38fd98, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/8949b9b71e824fbe92ded10c9d3fc3e3, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9e122496683048e38e8dddb6aeb4a061] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=35.4 K 2024-11-27T21:16:02,588 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6f2587577194e24abd8bc9ddf38fd98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732742158568 2024-11-27T21:16:02,589 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8949b9b71e824fbe92ded10c9d3fc3e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732742159761 2024-11-27T21:16:02,589 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e122496683048e38e8dddb6aeb4a061, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732742160509 2024-11-27T21:16:02,598 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#C#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:02,599 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/6bcfc11665464d04a33f9900727d3bce is 50, key is test_row_0/C:col10/1732742160509/Put/seqid=0 2024-11-27T21:16:02,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742222599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:02,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742309_1485 (size=12391) 2024-11-27T21:16:02,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742222620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742222621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:02,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742222621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:02,637 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/1579f6ca0e2645b49a76f7dd6583ec2d 2024-11-27T21:16:02,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/5397eebf1a534d91bff2feaca0297c44 is 50, key is test_row_1/C:col10/1732742161712/Put/seqid=0 2024-11-27T21:16:02,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742310_1486 (size=9757) 2024-11-27T21:16:02,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742222729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,739 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in cf3e016fc40bf5fab56083bda0e72b03 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:16:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:02,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742222808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:02,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742222839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742222839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:02,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:02,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742222849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:02,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:02,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:03,038 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/6bcfc11665464d04a33f9900727d3bce as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/6bcfc11665464d04a33f9900727d3bce 2024-11-27T21:16:03,042 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/C of cf3e016fc40bf5fab56083bda0e72b03 into 6bcfc11665464d04a33f9900727d3bce(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:03,042 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:03,042 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/C, priority=13, startTime=1732742162140; duration=0sec 2024-11-27T21:16:03,042 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:03,042 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:C 2024-11-27T21:16:03,055 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/5397eebf1a534d91bff2feaca0297c44 2024-11-27T21:16:03,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/c18d76137a974c2fb05a0638fac31d8b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/c18d76137a974c2fb05a0638fac31d8b 2024-11-27T21:16:03,062 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/c18d76137a974c2fb05a0638fac31d8b, entries=100, sequenceid=154, filesize=9.5 K 2024-11-27T21:16:03,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/1579f6ca0e2645b49a76f7dd6583ec2d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/1579f6ca0e2645b49a76f7dd6583ec2d 2024-11-27T21:16:03,066 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/1579f6ca0e2645b49a76f7dd6583ec2d, entries=100, sequenceid=154, filesize=9.5 K 2024-11-27T21:16:03,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/5397eebf1a534d91bff2feaca0297c44 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/5397eebf1a534d91bff2feaca0297c44 2024-11-27T21:16:03,069 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/5397eebf1a534d91bff2feaca0297c44, entries=100, sequenceid=154, filesize=9.5 K 2024-11-27T21:16:03,070 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cf3e016fc40bf5fab56083bda0e72b03 in 852ms, sequenceid=154, compaction requested=false 2024-11-27T21:16:03,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:03,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:03,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-27T21:16:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-27T21:16:03,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-27T21:16:03,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6900 sec 2024-11-27T21:16:03,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.6940 sec 2024-11-27T21:16:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:03,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-27T21:16:03,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:03,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:03,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:03,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:03,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:03,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:03,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/2443311d0db24dda9b8cbbb322a7bfe0 is 50, key is test_row_0/A:col10/1732742162491/Put/seqid=0 2024-11-27T21:16:03,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742311_1487 (size=14541) 2024-11-27T21:16:03,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742223160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:03,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:03,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742223161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:03,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:03,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:03,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742223166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:03,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:03,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742223169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:03,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:03,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742223279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:03,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742223279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:03,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742223279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:03,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742223489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:03,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:03,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742223489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:03,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:03,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:03,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742223489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:03,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:03,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/2443311d0db24dda9b8cbbb322a7bfe0 2024-11-27T21:16:03,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/d83636f25a364144a9ec4eb2973af953 is 50, key is test_row_0/B:col10/1732742162491/Put/seqid=0 2024-11-27T21:16:03,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742312_1488 (size=12151) 2024-11-27T21:16:03,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742223688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:03,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:03,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742223808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:03,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742223809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742223809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:03,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:03,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:03,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/d83636f25a364144a9ec4eb2973af953 2024-11-27T21:16:03,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/0671fb0e7cf7460092f5c7449de4c4bd is 50, key is test_row_0/C:col10/1732742162491/Put/seqid=0 2024-11-27T21:16:03,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742313_1489 (size=12151) 2024-11-27T21:16:03,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/0671fb0e7cf7460092f5c7449de4c4bd 2024-11-27T21:16:03,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/2443311d0db24dda9b8cbbb322a7bfe0 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/2443311d0db24dda9b8cbbb322a7bfe0 2024-11-27T21:16:03,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/2443311d0db24dda9b8cbbb322a7bfe0, entries=200, sequenceid=193, filesize=14.2 K 2024-11-27T21:16:03,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/d83636f25a364144a9ec4eb2973af953 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d83636f25a364144a9ec4eb2973af953 2024-11-27T21:16:03,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d83636f25a364144a9ec4eb2973af953, entries=150, sequenceid=193, filesize=11.9 K 2024-11-27T21:16:03,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/0671fb0e7cf7460092f5c7449de4c4bd as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/0671fb0e7cf7460092f5c7449de4c4bd 2024-11-27T21:16:03,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/0671fb0e7cf7460092f5c7449de4c4bd, entries=150, sequenceid=193, filesize=11.9 K 2024-11-27T21:16:03,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for cf3e016fc40bf5fab56083bda0e72b03 in 849ms, sequenceid=193, compaction requested=true 2024-11-27T21:16:03,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:03,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:16:03,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:03,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:16:03,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:03,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:16:03,984 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:03,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:03,984 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:03,984 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36689 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:03,985 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/A is initiating minor compaction (all files) 2024-11-27T21:16:03,985 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:03,985 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/A in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:03,985 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/B is initiating minor compaction (all files) 2024-11-27T21:16:03,985 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/fe0903bf936f4db9b1f2678028ae42f0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/c18d76137a974c2fb05a0638fac31d8b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/2443311d0db24dda9b8cbbb322a7bfe0] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=35.8 K 2024-11-27T21:16:03,985 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/B in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:03,985 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/5ab27b9e698d462a90c5a8ccb0622905, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/1579f6ca0e2645b49a76f7dd6583ec2d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d83636f25a364144a9ec4eb2973af953] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=33.5 K 2024-11-27T21:16:03,985 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ab27b9e698d462a90c5a8ccb0622905, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732742160509 2024-11-27T21:16:03,985 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe0903bf936f4db9b1f2678028ae42f0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732742160509 2024-11-27T21:16:03,985 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting c18d76137a974c2fb05a0638fac31d8b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732742161712 2024-11-27T21:16:03,985 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 1579f6ca0e2645b49a76f7dd6583ec2d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732742161712 2024-11-27T21:16:03,986 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2443311d0db24dda9b8cbbb322a7bfe0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732742162473 2024-11-27T21:16:03,986 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting d83636f25a364144a9ec4eb2973af953, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732742162482 2024-11-27T21:16:03,999 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#A#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:03,999 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#B#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:03,999 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/7c1a44383a7a4ee7975df8e61fccb666 is 50, key is test_row_0/B:col10/1732742162491/Put/seqid=0 2024-11-27T21:16:03,999 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/65c5aa4063f94b4783c5f577223f0d5e is 50, key is test_row_0/A:col10/1732742162491/Put/seqid=0 2024-11-27T21:16:04,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742314_1490 (size=12493) 2024-11-27T21:16:04,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742315_1491 (size=12493) 2024-11-27T21:16:04,029 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/65c5aa4063f94b4783c5f577223f0d5e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/65c5aa4063f94b4783c5f577223f0d5e 2024-11-27T21:16:04,032 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/7c1a44383a7a4ee7975df8e61fccb666 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/7c1a44383a7a4ee7975df8e61fccb666 2024-11-27T21:16:04,034 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/A of cf3e016fc40bf5fab56083bda0e72b03 into 65c5aa4063f94b4783c5f577223f0d5e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:04,034 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:04,034 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/A, priority=13, startTime=1732742163984; duration=0sec 2024-11-27T21:16:04,034 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:04,034 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:A 2024-11-27T21:16:04,034 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:04,035 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:04,035 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/C is initiating minor compaction (all files) 2024-11-27T21:16:04,035 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/C in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:04,035 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/6bcfc11665464d04a33f9900727d3bce, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/5397eebf1a534d91bff2feaca0297c44, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/0671fb0e7cf7460092f5c7449de4c4bd] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=33.5 K 2024-11-27T21:16:04,036 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bcfc11665464d04a33f9900727d3bce, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732742160509 2024-11-27T21:16:04,036 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5397eebf1a534d91bff2feaca0297c44, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732742161712 2024-11-27T21:16:04,037 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/B of cf3e016fc40bf5fab56083bda0e72b03 into 7c1a44383a7a4ee7975df8e61fccb666(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:04,037 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:04,037 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/B, priority=13, startTime=1732742163984; duration=0sec 2024-11-27T21:16:04,038 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:04,038 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:B 2024-11-27T21:16:04,039 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0671fb0e7cf7460092f5c7449de4c4bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732742162482 2024-11-27T21:16:04,046 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#C#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:04,047 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/fd5898fd222145df9e8336f22bb2d905 is 50, key is test_row_0/C:col10/1732742162491/Put/seqid=0 2024-11-27T21:16:04,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742316_1492 (size=12493) 2024-11-27T21:16:04,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:04,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:16:04,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:04,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:04,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:04,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:04,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:04,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:04,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/206c47c4ff9146f88327dbab6a361f77 is 50, key is test_row_0/A:col10/1732742164321/Put/seqid=0 2024-11-27T21:16:04,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742317_1493 (size=14541) 2024-11-27T21:16:04,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742164366, max timestamp=1732742164383Num uniques 859; 2024-11-27T21:16:04,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:16:04,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742164366, max timestamp=1732742164383Num uniques 859; 2024-11-27T21:16:04,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742164366, max timestamp=1732742164383Num uniques 859; 2024-11-27T21:16:04,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:16:04,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:16:04,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742224439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:04,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:04,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:04,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742224441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:04,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:04,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742224442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:04,460 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/fd5898fd222145df9e8336f22bb2d905 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/fd5898fd222145df9e8336f22bb2d905 2024-11-27T21:16:04,463 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/C of cf3e016fc40bf5fab56083bda0e72b03 into fd5898fd222145df9e8336f22bb2d905(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:04,464 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:04,464 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/C, priority=13, startTime=1732742163984; duration=0sec 2024-11-27T21:16:04,464 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:04,464 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:C 2024-11-27T21:16:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T21:16:04,549 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:04,549 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:16:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-27T21:16:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T21:16:04,551 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:16:04,552 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:16:04,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:16:04,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742224549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:04,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:04,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:04,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742224559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:04,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:04,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742224569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T21:16:04,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T21:16:04,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:04,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:04,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:04,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:04,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:04,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742224698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:04,712 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_0' in cf3e016fc40bf5fab56083bda0e72b03 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:16:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:04,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/206c47c4ff9146f88327dbab6a361f77 2024-11-27T21:16:04,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/66ba9f28faa442f9aca8bbe895c217bd is 50, key is test_row_0/B:col10/1732742164321/Put/seqid=0 2024-11-27T21:16:04,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742318_1494 (size=12151) 2024-11-27T21:16:04,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/66ba9f28faa442f9aca8bbe895c217bd 2024-11-27T21:16:04,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/840f5cda68874cc5acdc4579f6b99a37 is 50, key is test_row_0/C:col10/1732742164321/Put/seqid=0 2024-11-27T21:16:04,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742319_1495 (size=12151) 2024-11-27T21:16:04,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:04,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742224759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:04,770 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in cf3e016fc40bf5fab56083bda0e72b03 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:16:04,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:04,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:04,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742224769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:04,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:04,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:04,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742224779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:04,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742224789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:04,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T21:16:04,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:04,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:04,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:04,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:04,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:04,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:04,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T21:16:05,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T21:16:05,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:05,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:05,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:05,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:05,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:05,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742225089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:05,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:05,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742225099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:05,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742225109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:05,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/840f5cda68874cc5acdc4579f6b99a37 2024-11-27T21:16:05,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/206c47c4ff9146f88327dbab6a361f77 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/206c47c4ff9146f88327dbab6a361f77 2024-11-27T21:16:05,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/206c47c4ff9146f88327dbab6a361f77, entries=200, sequenceid=205, filesize=14.2 K 2024-11-27T21:16:05,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/66ba9f28faa442f9aca8bbe895c217bd as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/66ba9f28faa442f9aca8bbe895c217bd 2024-11-27T21:16:05,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/66ba9f28faa442f9aca8bbe895c217bd, entries=150, sequenceid=205, filesize=11.9 K 2024-11-27T21:16:05,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/840f5cda68874cc5acdc4579f6b99a37 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/840f5cda68874cc5acdc4579f6b99a37 2024-11-27T21:16:05,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T21:16:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:05,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:05,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/840f5cda68874cc5acdc4579f6b99a37, entries=150, sequenceid=205, filesize=11.9 K 2024-11-27T21:16:05,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cf3e016fc40bf5fab56083bda0e72b03 in 844ms, sequenceid=205, compaction requested=false 2024-11-27T21:16:05,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T21:16:05,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T21:16:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:05,317 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-27T21:16:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:05,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/e9ad557cb5244e999a1f699674cb4a40 is 50, key is test_row_0/A:col10/1732742164438/Put/seqid=0 2024-11-27T21:16:05,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742320_1496 (size=12151) 2024-11-27T21:16:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:05,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:05,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742225656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:05,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742225670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:05,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742225671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:05,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T21:16:05,728 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/e9ad557cb5244e999a1f699674cb4a40 2024-11-27T21:16:05,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/077b9ce8f37647b2953aeb812c260b4c is 50, key is test_row_0/B:col10/1732742164438/Put/seqid=0 2024-11-27T21:16:05,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742321_1497 (size=12151) 2024-11-27T21:16:05,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742225779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:05,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742225789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:05,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742225789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:06,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742225999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:06,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:06,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742226009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:06,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742226009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:06,139 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/077b9ce8f37647b2953aeb812c260b4c 2024-11-27T21:16:06,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/9b4e5e97ecad437bace500f492c7bb56 is 50, key is test_row_0/C:col10/1732742164438/Put/seqid=0 2024-11-27T21:16:06,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742322_1498 (size=12151) 2024-11-27T21:16:06,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742226319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:06,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742226329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:06,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742226329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:06,552 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/9b4e5e97ecad437bace500f492c7bb56 2024-11-27T21:16:06,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/e9ad557cb5244e999a1f699674cb4a40 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/e9ad557cb5244e999a1f699674cb4a40 2024-11-27T21:16:06,559 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/e9ad557cb5244e999a1f699674cb4a40, entries=150, sequenceid=242, filesize=11.9 K 2024-11-27T21:16:06,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/077b9ce8f37647b2953aeb812c260b4c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/077b9ce8f37647b2953aeb812c260b4c 2024-11-27T21:16:06,562 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/077b9ce8f37647b2953aeb812c260b4c, entries=150, sequenceid=242, filesize=11.9 K 2024-11-27T21:16:06,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/9b4e5e97ecad437bace500f492c7bb56 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9b4e5e97ecad437bace500f492c7bb56 2024-11-27T21:16:06,565 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9b4e5e97ecad437bace500f492c7bb56, entries=150, sequenceid=242, filesize=11.9 K 2024-11-27T21:16:06,566 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for cf3e016fc40bf5fab56083bda0e72b03 in 1249ms, sequenceid=242, compaction requested=true 2024-11-27T21:16:06,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:06,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:06,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-27T21:16:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-27T21:16:06,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-27T21:16:06,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0160 sec 2024-11-27T21:16:06,571 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.0210 sec 2024-11-27T21:16:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T21:16:06,699 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:06,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:16:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-27T21:16:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T21:16:06,701 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:16:06,702 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:16:06,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:16:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:06,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-27T21:16:06,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:06,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:06,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:06,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:06,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:06,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:06,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/bd5f26c9478a49cab2ad713530e55ac2 is 50, key is test_row_0/A:col10/1732742166733/Put/seqid=0 2024-11-27T21:16:06,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742323_1499 (size=14541) 2024-11-27T21:16:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T21:16:06,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T21:16:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:06,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:06,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:06,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:06,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742166804, max timestamp=1732742166856Num uniques 859; 2024-11-27T21:16:06,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:06,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742166804, max timestamp=1732742166856Num uniques 859; 2024-11-27T21:16:06,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:06,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742166804, max timestamp=1732742166856Num uniques 859; 2024-11-27T21:16:06,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:06,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742226915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:06,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742226915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:06,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:06,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:06,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742226917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:06,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:06,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742226917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:06,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:06,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:06,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:07,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T21:16:07,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:07,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T21:16:07,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742227019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:07,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:07,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742227019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:07,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:07,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742227048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742227049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:07,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/bd5f26c9478a49cab2ad713530e55ac2 2024-11-27T21:16:07,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/3ecdf921f2eb44978f8069aaa63cf3dc is 50, key is test_row_0/B:col10/1732742166733/Put/seqid=0 2024-11-27T21:16:07,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742324_1500 (size=12151) 2024-11-27T21:16:07,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T21:16:07,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:07,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:07,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:07,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742227239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:07,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742227239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:07,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742227268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:07,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742227269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:07,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T21:16:07,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:07,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:07,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:07,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T21:16:07,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T21:16:07,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:07,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:07,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:07,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/3ecdf921f2eb44978f8069aaa63cf3dc 2024-11-27T21:16:07,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/bcb71a7b770741b0bd247779c7379ee6 is 50, key is test_row_0/C:col10/1732742166733/Put/seqid=0 2024-11-27T21:16:07,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742325_1501 (size=12151) 2024-11-27T21:16:07,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742227559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:07,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:07,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742227559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:07,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742227588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:07,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742227589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:07,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T21:16:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:07,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T21:16:07,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:07,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:07,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:07,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T21:16:07,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T21:16:07,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:07,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:07,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:07,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:07,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/bcb71a7b770741b0bd247779c7379ee6 2024-11-27T21:16:07,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/bd5f26c9478a49cab2ad713530e55ac2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/bd5f26c9478a49cab2ad713530e55ac2 2024-11-27T21:16:07,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/bd5f26c9478a49cab2ad713530e55ac2, entries=200, sequenceid=253, filesize=14.2 K 2024-11-27T21:16:07,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/3ecdf921f2eb44978f8069aaa63cf3dc as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/3ecdf921f2eb44978f8069aaa63cf3dc 2024-11-27T21:16:07,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/3ecdf921f2eb44978f8069aaa63cf3dc, entries=150, sequenceid=253, filesize=11.9 K 2024-11-27T21:16:07,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/bcb71a7b770741b0bd247779c7379ee6 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/bcb71a7b770741b0bd247779c7379ee6 2024-11-27T21:16:07,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/bcb71a7b770741b0bd247779c7379ee6, entries=150, sequenceid=253, filesize=11.9 K 2024-11-27T21:16:07,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for cf3e016fc40bf5fab56083bda0e72b03 in 1248ms, sequenceid=253, compaction requested=true 2024-11-27T21:16:07,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:07,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:16:07,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:07,982 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:07,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:16:07,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:07,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:16:07,983 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:07,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:07,984 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:07,984 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/A is initiating minor compaction (all files) 2024-11-27T21:16:07,984 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:07,984 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/A in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:07,984 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/B is initiating minor compaction (all files) 2024-11-27T21:16:07,984 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/65c5aa4063f94b4783c5f577223f0d5e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/206c47c4ff9146f88327dbab6a361f77, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/e9ad557cb5244e999a1f699674cb4a40, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/bd5f26c9478a49cab2ad713530e55ac2] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=52.5 K 2024-11-27T21:16:07,984 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/B in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:07,984 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/7c1a44383a7a4ee7975df8e61fccb666, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/66ba9f28faa442f9aca8bbe895c217bd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/077b9ce8f37647b2953aeb812c260b4c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/3ecdf921f2eb44978f8069aaa63cf3dc] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=47.8 K 2024-11-27T21:16:07,984 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65c5aa4063f94b4783c5f577223f0d5e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732742162482 2024-11-27T21:16:07,984 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c1a44383a7a4ee7975df8e61fccb666, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732742162482 2024-11-27T21:16:07,985 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 206c47c4ff9146f88327dbab6a361f77, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732742163159 2024-11-27T21:16:07,985 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 66ba9f28faa442f9aca8bbe895c217bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732742163159 2024-11-27T21:16:07,985 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9ad557cb5244e999a1f699674cb4a40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732742164437 2024-11-27T21:16:07,985 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 077b9ce8f37647b2953aeb812c260b4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732742164437 2024-11-27T21:16:07,985 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd5f26c9478a49cab2ad713530e55ac2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732742165654 2024-11-27T21:16:07,985 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ecdf921f2eb44978f8069aaa63cf3dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732742165654 2024-11-27T21:16:07,994 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#A#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:07,994 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/1456e010e1be4eb5987317e02be86349 is 50, key is test_row_0/A:col10/1732742166733/Put/seqid=0 2024-11-27T21:16:07,997 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#B#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:07,997 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/6e9a6112e0e24284937cecb50d922d12 is 50, key is test_row_0/B:col10/1732742166733/Put/seqid=0 2024-11-27T21:16:08,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742326_1502 (size=12629) 2024-11-27T21:16:08,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742327_1503 (size=12629) 2024-11-27T21:16:08,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T21:16:08,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:08,078 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-27T21:16:08,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:08,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:08,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:08,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:08,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:08,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:08,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/087be4b323ff41609876eb29e216ce80 is 50, key is test_row_0/A:col10/1732742166914/Put/seqid=0 2024-11-27T21:16:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:08,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:08,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742328_1504 (size=12301) 2024-11-27T21:16:08,100 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/087be4b323ff41609876eb29e216ce80 2024-11-27T21:16:08,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/8e7fe7cc7b3b4fe0a219d3087ef7c346 is 50, key is test_row_0/B:col10/1732742166914/Put/seqid=0 2024-11-27T21:16:08,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742329_1505 (size=12301) 2024-11-27T21:16:08,117 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/8e7fe7cc7b3b4fe0a219d3087ef7c346 2024-11-27T21:16:08,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/dec703e6b70e4582bc99ad7cbe2e5a7f is 50, key is test_row_0/C:col10/1732742166914/Put/seqid=0 2024-11-27T21:16:08,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742228121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,128 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,128 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,128 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742330_1506 (size=12301) 2024-11-27T21:16:08,130 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/dec703e6b70e4582bc99ad7cbe2e5a7f 2024-11-27T21:16:08,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/087be4b323ff41609876eb29e216ce80 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/087be4b323ff41609876eb29e216ce80 2024-11-27T21:16:08,137 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/087be4b323ff41609876eb29e216ce80, entries=150, sequenceid=288, filesize=12.0 K 2024-11-27T21:16:08,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/8e7fe7cc7b3b4fe0a219d3087ef7c346 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8e7fe7cc7b3b4fe0a219d3087ef7c346 2024-11-27T21:16:08,140 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8e7fe7cc7b3b4fe0a219d3087ef7c346, entries=150, sequenceid=288, filesize=12.0 K 2024-11-27T21:16:08,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/dec703e6b70e4582bc99ad7cbe2e5a7f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/dec703e6b70e4582bc99ad7cbe2e5a7f 2024-11-27T21:16:08,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742228123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742228124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742228125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,146 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/dec703e6b70e4582bc99ad7cbe2e5a7f, entries=150, sequenceid=288, filesize=12.0 K 2024-11-27T21:16:08,146 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for cf3e016fc40bf5fab56083bda0e72b03 in 68ms, sequenceid=288, compaction requested=true 2024-11-27T21:16:08,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:08,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:08,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-27T21:16:08,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-27T21:16:08,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-27T21:16:08,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4460 sec 2024-11-27T21:16:08,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.4520 sec 2024-11-27T21:16:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:08,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-27T21:16:08,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:08,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:08,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:08,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:08,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:08,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:08,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/38fafd8f42a54d0984cb2392250cac4c is 50, key is test_row_0/A:col10/1732742168123/Put/seqid=0 2024-11-27T21:16:08,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742331_1507 (size=14741) 2024-11-27T21:16:08,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742168290, max timestamp=1732742168309Num uniques 859; 2024-11-27T21:16:08,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:08,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742168290, max timestamp=1732742168309Num uniques 859; 2024-11-27T21:16:08,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:08,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742168290, max timestamp=1732742168309Num uniques 859; 2024-11-27T21:16:08,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:08,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742228357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742228358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742228359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742228366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,407 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/1456e010e1be4eb5987317e02be86349 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/1456e010e1be4eb5987317e02be86349 2024-11-27T21:16:08,411 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/6e9a6112e0e24284937cecb50d922d12 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6e9a6112e0e24284937cecb50d922d12 2024-11-27T21:16:08,412 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/A of cf3e016fc40bf5fab56083bda0e72b03 into 1456e010e1be4eb5987317e02be86349(size=12.3 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:08,412 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:08,412 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/A, priority=12, startTime=1732742167982; duration=0sec 2024-11-27T21:16:08,412 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:08,412 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:A 2024-11-27T21:16:08,412 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T21:16:08,413 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61247 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T21:16:08,413 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/C is initiating minor compaction (all files) 2024-11-27T21:16:08,413 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/C in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:08,414 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/fd5898fd222145df9e8336f22bb2d905, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/840f5cda68874cc5acdc4579f6b99a37, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9b4e5e97ecad437bace500f492c7bb56, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/bcb71a7b770741b0bd247779c7379ee6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/dec703e6b70e4582bc99ad7cbe2e5a7f] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=59.8 K 2024-11-27T21:16:08,414 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd5898fd222145df9e8336f22bb2d905, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732742162482 2024-11-27T21:16:08,415 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 840f5cda68874cc5acdc4579f6b99a37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732742163159 2024-11-27T21:16:08,415 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b4e5e97ecad437bace500f492c7bb56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732742164437 2024-11-27T21:16:08,415 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/B of cf3e016fc40bf5fab56083bda0e72b03 into 6e9a6112e0e24284937cecb50d922d12(size=12.3 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:08,416 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:08,416 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/B, priority=12, startTime=1732742167982; duration=0sec 2024-11-27T21:16:08,416 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcb71a7b770741b0bd247779c7379ee6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732742165654 2024-11-27T21:16:08,416 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:08,416 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:B 2024-11-27T21:16:08,416 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting dec703e6b70e4582bc99ad7cbe2e5a7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732742166913 2024-11-27T21:16:08,428 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#C#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:08,428 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/6e17242580944a2b95e4193b829c1d97 is 50, key is test_row_0/C:col10/1732742166914/Put/seqid=0 2024-11-27T21:16:08,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742332_1508 (size=12813) 2024-11-27T21:16:08,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742228479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742228479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742228479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742228479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/38fafd8f42a54d0984cb2392250cac4c 2024-11-27T21:16:08,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/f518f934cd84487aaa7f5f5c0d11052a is 50, key is test_row_0/B:col10/1732742168123/Put/seqid=0 2024-11-27T21:16:08,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742333_1509 (size=12301) 2024-11-27T21:16:08,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742228699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742228699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742228699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742228700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1732742228809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:08,814 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_2' in cf3e016fc40bf5fab56083bda0e72b03 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8300 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:16:08,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:08,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:08,835 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/6e17242580944a2b95e4193b829c1d97 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/6e17242580944a2b95e4193b829c1d97 2024-11-27T21:16:08,839 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/C of cf3e016fc40bf5fab56083bda0e72b03 into 6e17242580944a2b95e4193b829c1d97(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:08,839 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:08,839 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/C, priority=11, startTime=1732742167983; duration=0sec 2024-11-27T21:16:08,839 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:08,839 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:C 2024-11-27T21:16:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T21:16:08,849 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:08,850 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:16:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-27T21:16:08,851 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:16:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T21:16:08,852 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:16:08,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:16:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T21:16:09,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:16:09,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:09,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:09,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:09,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742229019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:09,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742229019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:09,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742229019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:09,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742229019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:09,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/f518f934cd84487aaa7f5f5c0d11052a 2024-11-27T21:16:09,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/a98765a9276140579ecae6d79c5fef3e is 50, key is test_row_0/C:col10/1732742168123/Put/seqid=0 2024-11-27T21:16:09,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742334_1510 (size=12301) 2024-11-27T21:16:09,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:16:09,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:09,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:09,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:09,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T21:16:09,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:16:09,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:09,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:09,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:09,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:16:09,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:09,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:09,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:09,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/a98765a9276140579ecae6d79c5fef3e 2024-11-27T21:16:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T21:16:09,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/38fafd8f42a54d0984cb2392250cac4c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/38fafd8f42a54d0984cb2392250cac4c 2024-11-27T21:16:09,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/38fafd8f42a54d0984cb2392250cac4c, entries=200, sequenceid=299, filesize=14.4 K 2024-11-27T21:16:09,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/f518f934cd84487aaa7f5f5c0d11052a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/f518f934cd84487aaa7f5f5c0d11052a 2024-11-27T21:16:09,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/f518f934cd84487aaa7f5f5c0d11052a, entries=150, sequenceid=299, filesize=12.0 K 2024-11-27T21:16:09,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/a98765a9276140579ecae6d79c5fef3e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a98765a9276140579ecae6d79c5fef3e 2024-11-27T21:16:09,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a98765a9276140579ecae6d79c5fef3e, entries=150, sequenceid=299, filesize=12.0 K 2024-11-27T21:16:09,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cf3e016fc40bf5fab56083bda0e72b03 in 1242ms, sequenceid=299, compaction requested=true 2024-11-27T21:16:09,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:09,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:16:09,490 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:09,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:09,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:16:09,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:09,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:16:09,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:09,490 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:09,491 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:09,491 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/A is initiating minor compaction (all files) 2024-11-27T21:16:09,491 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:09,491 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/A in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:09,491 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/B is initiating minor compaction (all files) 2024-11-27T21:16:09,491 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/B in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:09,491 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/1456e010e1be4eb5987317e02be86349, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/087be4b323ff41609876eb29e216ce80, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/38fafd8f42a54d0984cb2392250cac4c] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=38.7 K 2024-11-27T21:16:09,491 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6e9a6112e0e24284937cecb50d922d12, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8e7fe7cc7b3b4fe0a219d3087ef7c346, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/f518f934cd84487aaa7f5f5c0d11052a] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=36.4 K 2024-11-27T21:16:09,492 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1456e010e1be4eb5987317e02be86349, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732742165654 2024-11-27T21:16:09,492 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e9a6112e0e24284937cecb50d922d12, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732742165654 2024-11-27T21:16:09,492 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 087be4b323ff41609876eb29e216ce80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732742166913 2024-11-27T21:16:09,492 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e7fe7cc7b3b4fe0a219d3087ef7c346, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732742166913 2024-11-27T21:16:09,492 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38fafd8f42a54d0984cb2392250cac4c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732742168100 2024-11-27T21:16:09,492 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting f518f934cd84487aaa7f5f5c0d11052a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732742168120 2024-11-27T21:16:09,498 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#B#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:09,498 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/81fc25db0f0f45129ec3c90f96f80c7d is 50, key is test_row_0/B:col10/1732742168123/Put/seqid=0 2024-11-27T21:16:09,501 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#A#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:09,501 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/c11d40c95895430ab0a884a44ffad0b5 is 50, key is test_row_0/A:col10/1732742168123/Put/seqid=0 2024-11-27T21:16:09,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742335_1511 (size=12881) 2024-11-27T21:16:09,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742336_1512 (size=12881) 2024-11-27T21:16:09,512 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/81fc25db0f0f45129ec3c90f96f80c7d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/81fc25db0f0f45129ec3c90f96f80c7d 2024-11-27T21:16:09,520 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/B of cf3e016fc40bf5fab56083bda0e72b03 into 81fc25db0f0f45129ec3c90f96f80c7d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:09,520 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:09,521 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/B, priority=13, startTime=1732742169490; duration=0sec 2024-11-27T21:16:09,521 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:09,521 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:B 2024-11-27T21:16:09,521 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T21:16:09,521 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T21:16:09,521 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T21:16:09,521 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. because compaction request was cancelled 2024-11-27T21:16:09,521 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:C 2024-11-27T21:16:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:09,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-27T21:16:09,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:09,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:09,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:09,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:09,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:09,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:09,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/9c2692e43aae461da81aa03f38c1fe2a is 50, key is test_row_0/A:col10/1732742168355/Put/seqid=0 2024-11-27T21:16:09,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742337_1513 (size=14741) 2024-11-27T21:16:09,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742229555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:09,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742229556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:09,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:09,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742229556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:09,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:09,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:09,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742229558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:09,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:16:09,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:09,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:09,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:09,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742229669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:09,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:09,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742229679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742229679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:09,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742229689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:09,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:16:09,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:09,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:09,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:09,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742229889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:09,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742229899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:09,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742229899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:09,914 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/c11d40c95895430ab0a884a44ffad0b5 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/c11d40c95895430ab0a884a44ffad0b5 2024-11-27T21:16:09,918 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/A of cf3e016fc40bf5fab56083bda0e72b03 into c11d40c95895430ab0a884a44ffad0b5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:09,918 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:09,918 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/A, priority=13, startTime=1732742169490; duration=0sec 2024-11-27T21:16:09,918 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:09,918 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:A 2024-11-27T21:16:09,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:16:09,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:09,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:09,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:09,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:09,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742229909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:09,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:09,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:09,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:09,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/9c2692e43aae461da81aa03f38c1fe2a 2024-11-27T21:16:09,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/8a3268c93fb4451794aac5fee15777c4 is 50, key is test_row_0/B:col10/1732742168355/Put/seqid=0 2024-11-27T21:16:09,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742338_1514 (size=12301) 2024-11-27T21:16:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T21:16:10,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:16:10,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:10,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:10,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:10,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:10,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:10,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742230208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:10,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:16:10,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:10,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:10,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:10,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:10,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:10,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742230219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:10,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742230219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:10,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742230230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:10,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/8a3268c93fb4451794aac5fee15777c4 2024-11-27T21:16:10,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/52cbfab76b0749bd972d147c5a47f62a is 50, key is test_row_0/C:col10/1732742168355/Put/seqid=0 2024-11-27T21:16:10,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742339_1515 (size=12301) 2024-11-27T21:16:10,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:16:10,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:10,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:10,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:10,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:10,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:10,503 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T21:16:10,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:16:10,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:10,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:10,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:10,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:10,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:10,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:16:10,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:10,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:10,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:10,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:10,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:10,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742230729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:10,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742230739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:10,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742230739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:10,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742230748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:10,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/52cbfab76b0749bd972d147c5a47f62a 2024-11-27T21:16:10,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/9c2692e43aae461da81aa03f38c1fe2a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9c2692e43aae461da81aa03f38c1fe2a 2024-11-27T21:16:10,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9c2692e43aae461da81aa03f38c1fe2a, entries=200, sequenceid=338, filesize=14.4 K 2024-11-27T21:16:10,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/8a3268c93fb4451794aac5fee15777c4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8a3268c93fb4451794aac5fee15777c4 2024-11-27T21:16:10,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8a3268c93fb4451794aac5fee15777c4, entries=150, sequenceid=338, filesize=12.0 K 2024-11-27T21:16:10,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/52cbfab76b0749bd972d147c5a47f62a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/52cbfab76b0749bd972d147c5a47f62a 2024-11-27T21:16:10,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/52cbfab76b0749bd972d147c5a47f62a, entries=150, sequenceid=338, filesize=12.0 K 2024-11-27T21:16:10,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for cf3e016fc40bf5fab56083bda0e72b03 in 1256ms, sequenceid=338, compaction requested=true 2024-11-27T21:16:10,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:10,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:16:10,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:10,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:16:10,794 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T21:16:10,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:10,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:16:10,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:10,794 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T21:16:10,795 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T21:16:10,795 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T21:16:10,795 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T21:16:10,795 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T21:16:10,795 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. because compaction request was cancelled 2024-11-27T21:16:10,795 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. because compaction request was cancelled 2024-11-27T21:16:10,795 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:A 2024-11-27T21:16:10,795 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:B 2024-11-27T21:16:10,795 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:10,796 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:10,796 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/C is initiating minor compaction (all files) 2024-11-27T21:16:10,796 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/C in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:10,796 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/6e17242580944a2b95e4193b829c1d97, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a98765a9276140579ecae6d79c5fef3e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/52cbfab76b0749bd972d147c5a47f62a] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=36.5 K 2024-11-27T21:16:10,797 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e17242580944a2b95e4193b829c1d97, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732742166913 2024-11-27T21:16:10,797 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting a98765a9276140579ecae6d79c5fef3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732742168120 2024-11-27T21:16:10,797 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52cbfab76b0749bd972d147c5a47f62a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732742168355 2024-11-27T21:16:10,803 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#C#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:10,804 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/7d51487c6ac346e9ba7b3cca86abf956 is 50, key is test_row_0/C:col10/1732742168355/Put/seqid=0 2024-11-27T21:16:10,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742340_1516 (size=12915) 2024-11-27T21:16:10,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T21:16:10,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:10,841 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:16:10,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:10,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:10,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:10,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:10,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:10,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/47f1bd0399db46b5b17712499834a502 is 50, key is test_row_0/A:col10/1732742169555/Put/seqid=0 2024-11-27T21:16:10,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742341_1517 (size=12301) 2024-11-27T21:16:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T21:16:11,212 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/7d51487c6ac346e9ba7b3cca86abf956 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/7d51487c6ac346e9ba7b3cca86abf956 2024-11-27T21:16:11,217 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/C of cf3e016fc40bf5fab56083bda0e72b03 into 7d51487c6ac346e9ba7b3cca86abf956(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:11,217 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:11,217 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/C, priority=13, startTime=1732742170794; duration=0sec 2024-11-27T21:16:11,217 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:11,217 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:C 2024-11-27T21:16:11,250 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/47f1bd0399db46b5b17712499834a502 2024-11-27T21:16:11,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/879ace997e7e4380b8dc17aa7d3c2206 is 50, key is test_row_0/B:col10/1732742169555/Put/seqid=0 2024-11-27T21:16:11,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742342_1518 (size=12301) 2024-11-27T21:16:11,662 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/879ace997e7e4380b8dc17aa7d3c2206 2024-11-27T21:16:11,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/64cbee97c7bc4e1988b80abe86b5d2f2 is 50, key is test_row_0/C:col10/1732742169555/Put/seqid=0 2024-11-27T21:16:11,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742343_1519 (size=12301) 2024-11-27T21:16:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:11,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:11,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742171808, max timestamp=1732742171846Num uniques 859; 2024-11-27T21:16:11,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742171808, max timestamp=1732742171846Num uniques 859; 2024-11-27T21:16:11,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:16:11,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:16:11,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742171808, max timestamp=1732742171846Num uniques 859; 2024-11-27T21:16:11,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:16:11,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742231902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:11,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:11,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:11,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:11,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742231904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:11,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:11,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:11,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:11,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742231906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:11,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742231906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742232009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742232028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742232029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742232029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,085 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/64cbee97c7bc4e1988b80abe86b5d2f2 2024-11-27T21:16:12,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/47f1bd0399db46b5b17712499834a502 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/47f1bd0399db46b5b17712499834a502 2024-11-27T21:16:12,091 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/47f1bd0399db46b5b17712499834a502, entries=150, sequenceid=349, filesize=12.0 K 2024-11-27T21:16:12,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/879ace997e7e4380b8dc17aa7d3c2206 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/879ace997e7e4380b8dc17aa7d3c2206 2024-11-27T21:16:12,095 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/879ace997e7e4380b8dc17aa7d3c2206, entries=150, sequenceid=349, filesize=12.0 K 2024-11-27T21:16:12,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/64cbee97c7bc4e1988b80abe86b5d2f2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/64cbee97c7bc4e1988b80abe86b5d2f2 2024-11-27T21:16:12,101 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/64cbee97c7bc4e1988b80abe86b5d2f2, entries=150, sequenceid=349, filesize=12.0 K 2024-11-27T21:16:12,101 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cf3e016fc40bf5fab56083bda0e72b03 in 1260ms, sequenceid=349, compaction requested=true 2024-11-27T21:16:12,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:12,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:12,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-27T21:16:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-27T21:16:12,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-27T21:16:12,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2510 sec 2024-11-27T21:16:12,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 3.2550 sec 2024-11-27T21:16:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:12,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-27T21:16:12,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:12,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:16:12,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:12,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:16:12,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:12,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:16:12,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/4bd297486aad4ffdaef7451c8d5b3ce1 is 50, key is test_row_0/A:col10/1732742171900/Put/seqid=0 2024-11-27T21:16:12,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742344_1520 (size=14741) 2024-11-27T21:16:12,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742232261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742232262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742232265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742232268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742232389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742232389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742232389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742232390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742232599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742232599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742232599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742232600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/4bd297486aad4ffdaef7451c8d5b3ce1 2024-11-27T21:16:12,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/3a4afabceb2f49ff82911ee977d54299 is 50, key is test_row_0/B:col10/1732742171900/Put/seqid=0 2024-11-27T21:16:12,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742345_1521 (size=12301) 2024-11-27T21:16:12,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/3a4afabceb2f49ff82911ee977d54299 2024-11-27T21:16:12,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/0b7709f09c8b498c9f3d7756c12d5efe is 50, key is test_row_0/C:col10/1732742171900/Put/seqid=0 2024-11-27T21:16:12,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742346_1522 (size=12301) 2024-11-27T21:16:12,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742232919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742232919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742232920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742232920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T21:16:13,009 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:13,009 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:16:13,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-27T21:16:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T21:16:13,011 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:16:13,012 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:16:13,012 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:16:13,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/0b7709f09c8b498c9f3d7756c12d5efe 2024-11-27T21:16:13,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/4bd297486aad4ffdaef7451c8d5b3ce1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/4bd297486aad4ffdaef7451c8d5b3ce1 2024-11-27T21:16:13,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/4bd297486aad4ffdaef7451c8d5b3ce1, entries=200, sequenceid=388, filesize=14.4 K 2024-11-27T21:16:13,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/3a4afabceb2f49ff82911ee977d54299 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/3a4afabceb2f49ff82911ee977d54299 2024-11-27T21:16:13,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T21:16:13,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/3a4afabceb2f49ff82911ee977d54299, entries=150, sequenceid=388, filesize=12.0 K 2024-11-27T21:16:13,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/0b7709f09c8b498c9f3d7756c12d5efe as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/0b7709f09c8b498c9f3d7756c12d5efe 2024-11-27T21:16:13,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/0b7709f09c8b498c9f3d7756c12d5efe, entries=150, sequenceid=388, filesize=12.0 K 2024-11-27T21:16:13,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for cf3e016fc40bf5fab56083bda0e72b03 in 885ms, sequenceid=388, compaction requested=true 2024-11-27T21:16:13,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:13,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:16:13,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:13,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:16:13,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:13,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:16:13,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:13,125 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:13,125 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:13,127 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54664 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:13,127 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/A is initiating minor compaction (all files) 2024-11-27T21:16:13,127 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/A in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:13,127 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:13,127 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/B is initiating minor compaction (all files) 2024-11-27T21:16:13,127 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/c11d40c95895430ab0a884a44ffad0b5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9c2692e43aae461da81aa03f38c1fe2a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/47f1bd0399db46b5b17712499834a502, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/4bd297486aad4ffdaef7451c8d5b3ce1] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=53.4 K 2024-11-27T21:16:13,127 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/B in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:13,127 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/81fc25db0f0f45129ec3c90f96f80c7d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8a3268c93fb4451794aac5fee15777c4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/879ace997e7e4380b8dc17aa7d3c2206, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/3a4afabceb2f49ff82911ee977d54299] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=48.6 K 2024-11-27T21:16:13,128 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting c11d40c95895430ab0a884a44ffad0b5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732742168120 2024-11-27T21:16:13,128 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 81fc25db0f0f45129ec3c90f96f80c7d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732742168120 2024-11-27T21:16:13,128 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c2692e43aae461da81aa03f38c1fe2a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732742168355 2024-11-27T21:16:13,128 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a3268c93fb4451794aac5fee15777c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732742168355 2024-11-27T21:16:13,129 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 879ace997e7e4380b8dc17aa7d3c2206, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732742169553 2024-11-27T21:16:13,129 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47f1bd0399db46b5b17712499834a502, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732742169553 2024-11-27T21:16:13,129 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bd297486aad4ffdaef7451c8d5b3ce1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732742171900 2024-11-27T21:16:13,129 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a4afabceb2f49ff82911ee977d54299, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732742171900 2024-11-27T21:16:13,143 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#A#compaction#445 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:13,143 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#B#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:13,143 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/9fe529a4aea64dbbac5060e27e15c9b9 is 50, key is test_row_0/A:col10/1732742171900/Put/seqid=0 2024-11-27T21:16:13,144 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/b8ceb1d570fb46a2811d3c5fbcb1151b is 50, key is test_row_0/B:col10/1732742171900/Put/seqid=0 2024-11-27T21:16:13,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742347_1523 (size=13017) 2024-11-27T21:16:13,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742348_1524 (size=13017) 2024-11-27T21:16:13,162 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/9fe529a4aea64dbbac5060e27e15c9b9 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9fe529a4aea64dbbac5060e27e15c9b9 2024-11-27T21:16:13,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T21:16:13,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:13,164 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-27T21:16:13,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:13,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:13,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:13,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:13,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:13,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/da93eefc142342209b76739b9a28dc63 is 50, key is test_row_0/A:col10/1732742172260/Put/seqid=0 2024-11-27T21:16:13,171 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/A of cf3e016fc40bf5fab56083bda0e72b03 into 9fe529a4aea64dbbac5060e27e15c9b9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:13,171 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:13,171 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/A, priority=12, startTime=1732742173125; duration=0sec 2024-11-27T21:16:13,171 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:13,171 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:A 2024-11-27T21:16:13,172 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:13,174 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:13,174 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/C is initiating minor compaction (all files) 2024-11-27T21:16:13,174 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/C in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:13,174 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/7d51487c6ac346e9ba7b3cca86abf956, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/64cbee97c7bc4e1988b80abe86b5d2f2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/0b7709f09c8b498c9f3d7756c12d5efe] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=36.6 K 2024-11-27T21:16:13,175 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d51487c6ac346e9ba7b3cca86abf956, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732742168355 2024-11-27T21:16:13,175 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64cbee97c7bc4e1988b80abe86b5d2f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732742169553 2024-11-27T21:16:13,175 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b7709f09c8b498c9f3d7756c12d5efe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732742171900 2024-11-27T21:16:13,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742349_1525 (size=12301) 2024-11-27T21:16:13,185 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/da93eefc142342209b76739b9a28dc63 2024-11-27T21:16:13,195 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#C#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:13,195 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/9bb8d1456c0e4a89bb5983d32fedf257 is 50, key is test_row_0/C:col10/1732742171900/Put/seqid=0 2024-11-27T21:16:13,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/077e03dea9db450d857c99d0a6bbd3f1 is 50, key is test_row_0/B:col10/1732742172260/Put/seqid=0 2024-11-27T21:16:13,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742350_1526 (size=13017) 2024-11-27T21:16:13,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742351_1527 (size=12301) 2024-11-27T21:16:13,204 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/077e03dea9db450d857c99d0a6bbd3f1 2024-11-27T21:16:13,204 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/9bb8d1456c0e4a89bb5983d32fedf257 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9bb8d1456c0e4a89bb5983d32fedf257 2024-11-27T21:16:13,210 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/C of cf3e016fc40bf5fab56083bda0e72b03 into 9bb8d1456c0e4a89bb5983d32fedf257(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:13,210 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:13,210 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/C, priority=13, startTime=1732742173125; duration=0sec 2024-11-27T21:16:13,210 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:13,210 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:C 2024-11-27T21:16:13,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/d41fa4fd1d8841f6a7801c5cf37e9566 is 50, key is test_row_0/C:col10/1732742172260/Put/seqid=0 2024-11-27T21:16:13,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742352_1528 (size=12301) 2024-11-27T21:16:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T21:16:13,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:13,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:13,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742173455, max timestamp=1732742173480Num uniques 859; 2024-11-27T21:16:13,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:13,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742173455, max timestamp=1732742173480Num uniques 859; 2024-11-27T21:16:13,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742173455, max timestamp=1732742173480Num uniques 859; 2024-11-27T21:16:13,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:13,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:13,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742233522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:13,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742233523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:13,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742233530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:13,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742233531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:13,561 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/b8ceb1d570fb46a2811d3c5fbcb1151b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/b8ceb1d570fb46a2811d3c5fbcb1151b 2024-11-27T21:16:13,564 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/B of cf3e016fc40bf5fab56083bda0e72b03 into b8ceb1d570fb46a2811d3c5fbcb1151b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:13,564 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:13,564 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/B, priority=12, startTime=1732742173125; duration=0sec 2024-11-27T21:16:13,565 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:13,565 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:B 2024-11-27T21:16:13,615 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/d41fa4fd1d8841f6a7801c5cf37e9566 2024-11-27T21:16:13,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/da93eefc142342209b76739b9a28dc63 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/da93eefc142342209b76739b9a28dc63 2024-11-27T21:16:13,622 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/da93eefc142342209b76739b9a28dc63, entries=150, sequenceid=397, filesize=12.0 K 2024-11-27T21:16:13,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/077e03dea9db450d857c99d0a6bbd3f1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/077e03dea9db450d857c99d0a6bbd3f1 2024-11-27T21:16:13,626 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/077e03dea9db450d857c99d0a6bbd3f1, entries=150, sequenceid=397, filesize=12.0 K 2024-11-27T21:16:13,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/d41fa4fd1d8841f6a7801c5cf37e9566 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/d41fa4fd1d8841f6a7801c5cf37e9566 2024-11-27T21:16:13,630 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/d41fa4fd1d8841f6a7801c5cf37e9566, entries=150, sequenceid=397, filesize=12.0 K 2024-11-27T21:16:13,631 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for cf3e016fc40bf5fab56083bda0e72b03 in 467ms, sequenceid=397, compaction requested=false 2024-11-27T21:16:13,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:13,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:13,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-27T21:16:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-27T21:16:13,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-27T21:16:13,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 620 msec 2024-11-27T21:16:13,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 626 msec 2024-11-27T21:16:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T21:16:13,639 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:13,639 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:16:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-27T21:16:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T21:16:13,641 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:16:13,641 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:16:13,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:16:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:13,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-27T21:16:13,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:13,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:13,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:13,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:13,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:13,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:13,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/b3ab7979957f4bd58539d7a933d7fd6e is 50, key is test_row_0/A:col10/1732742173522/Put/seqid=0 2024-11-27T21:16:13,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742353_1529 (size=14741) 2024-11-27T21:16:13,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742233658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:13,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742233659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:13,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742233661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:13,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742233663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:13,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T21:16:13,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742233779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:13,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742233779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:13,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742233779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:13,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742233779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:13,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T21:16:13,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:13,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:13,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:13,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:13,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:13,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:13,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T21:16:13,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:13,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:13,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:13,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:13,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:13,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T21:16:14,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742233999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:14,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:14,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742233999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:14,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:14,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742233999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:14,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742233999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:14,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/b3ab7979957f4bd58539d7a933d7fd6e 2024-11-27T21:16:14,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/d805d205d8574c58a586dcf8e4e05385 is 50, key is test_row_0/B:col10/1732742173522/Put/seqid=0 2024-11-27T21:16:14,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742354_1530 (size=12301) 2024-11-27T21:16:14,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/d805d205d8574c58a586dcf8e4e05385 2024-11-27T21:16:14,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/09346b9085604b16a4c0ab6c43adac1a is 50, key is test_row_0/C:col10/1732742173522/Put/seqid=0 2024-11-27T21:16:14,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742355_1531 (size=12301) 2024-11-27T21:16:14,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T21:16:14,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:14,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:14,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:14,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:14,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:14,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T21:16:14,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:14,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:14,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:14,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:14,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T21:16:14,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742234319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:14,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742234319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742234319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:14,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742234319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:14,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T21:16:14,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:14,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:14,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:14,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:14,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:14,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/09346b9085604b16a4c0ab6c43adac1a 2024-11-27T21:16:14,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/b3ab7979957f4bd58539d7a933d7fd6e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/b3ab7979957f4bd58539d7a933d7fd6e 2024-11-27T21:16:14,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/b3ab7979957f4bd58539d7a933d7fd6e, entries=200, sequenceid=437, filesize=14.4 K 2024-11-27T21:16:14,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/d805d205d8574c58a586dcf8e4e05385 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d805d205d8574c58a586dcf8e4e05385 2024-11-27T21:16:14,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d805d205d8574c58a586dcf8e4e05385, entries=150, sequenceid=437, filesize=12.0 K 2024-11-27T21:16:14,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/09346b9085604b16a4c0ab6c43adac1a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/09346b9085604b16a4c0ab6c43adac1a 2024-11-27T21:16:14,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/09346b9085604b16a4c0ab6c43adac1a, entries=150, sequenceid=437, filesize=12.0 K 2024-11-27T21:16:14,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for cf3e016fc40bf5fab56083bda0e72b03 in 848ms, sequenceid=437, compaction requested=true 2024-11-27T21:16:14,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:14,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:16:14,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:14,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:16:14,498 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:14,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:14,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:16:14,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T21:16:14,499 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:14,499 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:14,499 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/A is initiating minor compaction (all files) 2024-11-27T21:16:14,499 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:14,499 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/A in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:14,499 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/B is initiating minor compaction (all files) 2024-11-27T21:16:14,499 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9fe529a4aea64dbbac5060e27e15c9b9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/da93eefc142342209b76739b9a28dc63, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/b3ab7979957f4bd58539d7a933d7fd6e] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=39.1 K 2024-11-27T21:16:14,499 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/B in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:14,500 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/b8ceb1d570fb46a2811d3c5fbcb1151b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/077e03dea9db450d857c99d0a6bbd3f1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d805d205d8574c58a586dcf8e4e05385] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=36.7 K 2024-11-27T21:16:14,500 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fe529a4aea64dbbac5060e27e15c9b9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732742171900 2024-11-27T21:16:14,500 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting b8ceb1d570fb46a2811d3c5fbcb1151b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732742171900 2024-11-27T21:16:14,500 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting da93eefc142342209b76739b9a28dc63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732742172260 2024-11-27T21:16:14,500 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 077e03dea9db450d857c99d0a6bbd3f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732742172260 2024-11-27T21:16:14,500 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3ab7979957f4bd58539d7a933d7fd6e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732742173519 2024-11-27T21:16:14,500 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting d805d205d8574c58a586dcf8e4e05385, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732742173522 2024-11-27T21:16:14,508 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#A#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:14,509 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/6a6c7640a53e40a5a4f5af65209cd38b is 50, key is test_row_0/A:col10/1732742173522/Put/seqid=0 2024-11-27T21:16:14,510 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#B#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:14,511 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/7ee51a22b0c84c83a432a3ce5f0e6c8e is 50, key is test_row_0/B:col10/1732742173522/Put/seqid=0 2024-11-27T21:16:14,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742356_1532 (size=13119) 2024-11-27T21:16:14,517 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/6a6c7640a53e40a5a4f5af65209cd38b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/6a6c7640a53e40a5a4f5af65209cd38b 2024-11-27T21:16:14,522 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/A of cf3e016fc40bf5fab56083bda0e72b03 into 6a6c7640a53e40a5a4f5af65209cd38b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:14,522 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:14,522 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/A, priority=13, startTime=1732742174498; duration=0sec 2024-11-27T21:16:14,522 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:14,522 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:A 2024-11-27T21:16:14,522 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:14,523 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:14,523 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/C is initiating minor compaction (all files) 2024-11-27T21:16:14,523 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/C in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:14,523 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9bb8d1456c0e4a89bb5983d32fedf257, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/d41fa4fd1d8841f6a7801c5cf37e9566, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/09346b9085604b16a4c0ab6c43adac1a] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=36.7 K 2024-11-27T21:16:14,523 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bb8d1456c0e4a89bb5983d32fedf257, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732742171900 2024-11-27T21:16:14,523 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting d41fa4fd1d8841f6a7801c5cf37e9566, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732742172260 2024-11-27T21:16:14,524 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09346b9085604b16a4c0ab6c43adac1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732742173522 2024-11-27T21:16:14,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742357_1533 (size=13119) 2024-11-27T21:16:14,531 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/7ee51a22b0c84c83a432a3ce5f0e6c8e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/7ee51a22b0c84c83a432a3ce5f0e6c8e 2024-11-27T21:16:14,531 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#C#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:14,531 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/1b83051fe39b453eb8e5da354abae6d3 is 50, key is test_row_0/C:col10/1732742173522/Put/seqid=0 2024-11-27T21:16:14,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742358_1534 (size=13119) 2024-11-27T21:16:14,537 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/B of cf3e016fc40bf5fab56083bda0e72b03 into 7ee51a22b0c84c83a432a3ce5f0e6c8e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:14,537 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:14,537 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/B, priority=13, startTime=1732742174498; duration=0sec 2024-11-27T21:16:14,537 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:14,537 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:B 2024-11-27T21:16:14,544 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/1b83051fe39b453eb8e5da354abae6d3 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/1b83051fe39b453eb8e5da354abae6d3 2024-11-27T21:16:14,548 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/C of cf3e016fc40bf5fab56083bda0e72b03 into 1b83051fe39b453eb8e5da354abae6d3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:14,548 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:14,548 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/C, priority=13, startTime=1732742174499; duration=0sec 2024-11-27T21:16:14,548 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:14,548 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:C 2024-11-27T21:16:14,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T21:16:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:14,558 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-27T21:16:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:14,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/a4c9f8aff66b4f8c82221b7370ced140 is 50, key is test_row_0/A:col10/1732742173655/Put/seqid=0 2024-11-27T21:16:14,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742359_1535 (size=12301) 2024-11-27T21:16:14,568 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/a4c9f8aff66b4f8c82221b7370ced140 2024-11-27T21:16:14,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/6e344eac0f914ce2a496d3b81fde118d is 50, key is test_row_0/B:col10/1732742173655/Put/seqid=0 2024-11-27T21:16:14,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742360_1536 (size=12301) 2024-11-27T21:16:14,581 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/6e344eac0f914ce2a496d3b81fde118d 2024-11-27T21:16:14,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/10720badecfc49c9a2dcf4dd88d0d263 is 50, key is test_row_0/C:col10/1732742173655/Put/seqid=0 2024-11-27T21:16:14,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742361_1537 (size=12301) 2024-11-27T21:16:14,602 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/10720badecfc49c9a2dcf4dd88d0d263 2024-11-27T21:16:14,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/a4c9f8aff66b4f8c82221b7370ced140 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/a4c9f8aff66b4f8c82221b7370ced140 2024-11-27T21:16:14,612 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/a4c9f8aff66b4f8c82221b7370ced140, entries=150, sequenceid=448, filesize=12.0 K 2024-11-27T21:16:14,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/6e344eac0f914ce2a496d3b81fde118d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6e344eac0f914ce2a496d3b81fde118d 2024-11-27T21:16:14,620 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6e344eac0f914ce2a496d3b81fde118d, entries=150, sequenceid=448, filesize=12.0 K 2024-11-27T21:16:14,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/10720badecfc49c9a2dcf4dd88d0d263 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/10720badecfc49c9a2dcf4dd88d0d263 2024-11-27T21:16:14,630 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/10720badecfc49c9a2dcf4dd88d0d263, entries=150, sequenceid=448, filesize=12.0 K 2024-11-27T21:16:14,631 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cf3e016fc40bf5fab56083bda0e72b03 in 73ms, sequenceid=448, compaction requested=false 2024-11-27T21:16:14,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:14,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:14,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-27T21:16:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-27T21:16:14,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-27T21:16:14,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 992 msec 2024-11-27T21:16:14,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 996 msec 2024-11-27T21:16:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T21:16:14,779 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:14,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:16:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-27T21:16:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T21:16:14,782 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:16:14,782 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:16:14,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:16:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:14,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:16:14,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:14,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:14,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:14,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:14,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:14,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:14,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/94f707994deb4502a00d7399519da55b is 50, key is test_row_0/A:col10/1732742174833/Put/seqid=0 2024-11-27T21:16:14,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742362_1538 (size=17181) 2024-11-27T21:16:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T21:16:14,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742174871, max timestamp=1732742174919Num uniques 859; 2024-11-27T21:16:14,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:16:14,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742174871, max timestamp=1732742174919Num uniques 859; 2024-11-27T21:16:14,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742174871, max timestamp=1732742174919Num uniques 859; 2024-11-27T21:16:14,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:16:14,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:16:14,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:14,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:14,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:14,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:14,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:14,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:14,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:14,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742234965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742234965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:14,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742234967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:14,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742234982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:15,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:15,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:15,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:15,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742235089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742235089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742235089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T21:16:15,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742235109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:15,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:15,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:15,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:15,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:15,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/94f707994deb4502a00d7399519da55b 2024-11-27T21:16:15,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/c1040852b5924c05ac972e294a7249d0 is 50, key is test_row_0/B:col10/1732742174833/Put/seqid=0 2024-11-27T21:16:15,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742363_1539 (size=12301) 2024-11-27T21:16:15,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742235299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:15,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742235299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:15,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742235299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:15,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742235319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:15,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:15,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:15,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:15,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:15,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:15,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T21:16:15,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:15,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:15,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:15,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:15,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57122 deadline: 1732742235619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:15,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57170 deadline: 1732742235619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:15,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57152 deadline: 1732742235619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:15,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57192 deadline: 1732742235638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 , the old value is region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., hostname=4585c39bed03,37381,1732742022265, seqNum=2 because the exception is null or not the one we care about 2024-11-27T21:16:15,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/c1040852b5924c05ac972e294a7249d0 2024-11-27T21:16:15,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/a52c9fc73bb5458e97cc909247d13f64 is 50, key is test_row_0/C:col10/1732742174833/Put/seqid=0 2024-11-27T21:16:15,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742364_1540 (size=12301) 2024-11-27T21:16:15,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:15,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:15,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:15,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:15,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:15,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:15,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:15,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:15,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T21:16:16,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:16,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:16,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/a52c9fc73bb5458e97cc909247d13f64 2024-11-27T21:16:16,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/94f707994deb4502a00d7399519da55b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/94f707994deb4502a00d7399519da55b 2024-11-27T21:16:16,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/94f707994deb4502a00d7399519da55b, entries=250, sequenceid=459, filesize=16.8 K 2024-11-27T21:16:16,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/c1040852b5924c05ac972e294a7249d0 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/c1040852b5924c05ac972e294a7249d0 2024-11-27T21:16:16,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/c1040852b5924c05ac972e294a7249d0, entries=150, sequenceid=459, filesize=12.0 K 2024-11-27T21:16:16,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/a52c9fc73bb5458e97cc909247d13f64 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a52c9fc73bb5458e97cc909247d13f64 2024-11-27T21:16:16,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a52c9fc73bb5458e97cc909247d13f64, entries=150, sequenceid=459, filesize=12.0 K 2024-11-27T21:16:16,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cf3e016fc40bf5fab56083bda0e72b03 in 1250ms, sequenceid=459, compaction requested=true 2024-11-27T21:16:16,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:16,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:16:16,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:16,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:16:16,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:16,091 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:16,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf3e016fc40bf5fab56083bda0e72b03:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:16:16,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:16,091 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:16,092 INFO [Thread-2068 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2068. 2024-11-27T21:16:16,092 DEBUG [Thread-2068 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:16,092 DEBUG [Thread-2068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,093 DEBUG [Thread-2068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,093 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:16,094 INFO [Thread-2076 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2076. 2024-11-27T21:16:16,094 DEBUG [Thread-2076 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:16,094 DEBUG [Thread-2076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,094 DEBUG [Thread-2076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,094 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:16,096 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:16,096 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42601 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:16,096 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/A is initiating minor compaction (all files) 2024-11-27T21:16:16,096 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/B is initiating minor compaction (all files) 2024-11-27T21:16:16,096 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/B in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,096 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/A in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,096 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/6a6c7640a53e40a5a4f5af65209cd38b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/a4c9f8aff66b4f8c82221b7370ced140, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/94f707994deb4502a00d7399519da55b] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=41.6 K 2024-11-27T21:16:16,096 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/7ee51a22b0c84c83a432a3ce5f0e6c8e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6e344eac0f914ce2a496d3b81fde118d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/c1040852b5924c05ac972e294a7249d0] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=36.8 K 2024-11-27T21:16:16,097 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a6c7640a53e40a5a4f5af65209cd38b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732742173522 2024-11-27T21:16:16,097 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ee51a22b0c84c83a432a3ce5f0e6c8e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732742173522 2024-11-27T21:16:16,097 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4c9f8aff66b4f8c82221b7370ced140, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732742173655 2024-11-27T21:16:16,097 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e344eac0f914ce2a496d3b81fde118d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732742173655 2024-11-27T21:16:16,097 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94f707994deb4502a00d7399519da55b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732742174830 2024-11-27T21:16:16,097 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting c1040852b5924c05ac972e294a7249d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732742174830 2024-11-27T21:16:16,098 INFO [Thread-2072 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2072. 2024-11-27T21:16:16,098 DEBUG [Thread-2072 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:16,099 DEBUG [Thread-2072 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:16,099 DEBUG [Thread-2072 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,103 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#B#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:16,103 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#A#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:16,103 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/4a5a923eb01745739950a09e3b6917cf is 50, key is test_row_0/B:col10/1732742174833/Put/seqid=0 2024-11-27T21:16:16,104 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/992a313d18ab45b8816c13ecd76b5a74 is 50, key is test_row_0/A:col10/1732742174833/Put/seqid=0 2024-11-27T21:16:16,104 INFO [Thread-2074 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2074. 2024-11-27T21:16:16,104 DEBUG [Thread-2074 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:16,104 DEBUG [Thread-2074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,105 DEBUG [Thread-2074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:16,105 INFO [Thread-2070 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2070. 2024-11-27T21:16:16,105 DEBUG [Thread-2070 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:16,105 DEBUG [Thread-2070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,105 DEBUG [Thread-2070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:16,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742365_1541 (size=13221) 2024-11-27T21:16:16,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742366_1542 (size=13221) 2024-11-27T21:16:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:16,143 INFO [Thread-2063 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2063. 2024-11-27T21:16:16,143 DEBUG [Thread-2063 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:16,143 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,143 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,143 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:16,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=147.60 KB heapSize=425.53 KB 2024-11-27T21:16:16,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:16,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:16:16,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:16,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:16:16,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:16,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:16:16,144 INFO [Thread-2059 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2059. 2024-11-27T21:16:16,144 DEBUG [Thread-2059 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:16,144 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,144 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,144 INFO [Thread-2057 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2057. 2024-11-27T21:16:16,144 DEBUG [Thread-2057 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:16,144 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,145 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,145 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:16,145 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:16,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/30864e07b47341e893796e4ab664df12 is 50, key is test_row_0/A:col10/1732742174980/Put/seqid=0 2024-11-27T21:16:16,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742367_1543 (size=12301) 2024-11-27T21:16:16,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:16,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:16,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,160 INFO [Thread-2065 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2065. 2024-11-27T21:16:16,160 DEBUG [Thread-2065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:16,160 DEBUG [Thread-2065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,160 DEBUG [Thread-2065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:16,161 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:16,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:16,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:16,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:16,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,514 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/4a5a923eb01745739950a09e3b6917cf as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/4a5a923eb01745739950a09e3b6917cf 2024-11-27T21:16:16,514 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/992a313d18ab45b8816c13ecd76b5a74 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/992a313d18ab45b8816c13ecd76b5a74 2024-11-27T21:16:16,517 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/B of cf3e016fc40bf5fab56083bda0e72b03 into 4a5a923eb01745739950a09e3b6917cf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:16,517 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/A of cf3e016fc40bf5fab56083bda0e72b03 into 992a313d18ab45b8816c13ecd76b5a74(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:16,517 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:16,517 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:16,518 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/B, priority=13, startTime=1732742176091; duration=0sec 2024-11-27T21:16:16,518 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/A, priority=13, startTime=1732742176090; duration=0sec 2024-11-27T21:16:16,518 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:16,518 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:B 2024-11-27T21:16:16,518 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:16,518 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:A 2024-11-27T21:16:16,518 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:16,518 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:16,518 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): cf3e016fc40bf5fab56083bda0e72b03/C is initiating minor compaction (all files) 2024-11-27T21:16:16,518 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf3e016fc40bf5fab56083bda0e72b03/C in TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,518 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/1b83051fe39b453eb8e5da354abae6d3, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/10720badecfc49c9a2dcf4dd88d0d263, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a52c9fc73bb5458e97cc909247d13f64] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp, totalSize=36.8 K 2024-11-27T21:16:16,519 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b83051fe39b453eb8e5da354abae6d3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732742173522 2024-11-27T21:16:16,519 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 10720badecfc49c9a2dcf4dd88d0d263, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732742173655 2024-11-27T21:16:16,519 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting a52c9fc73bb5458e97cc909247d13f64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732742174830 2024-11-27T21:16:16,524 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf3e016fc40bf5fab56083bda0e72b03#C#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:16,524 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/656c4f37d6b04f9d9bc85429947f7e15 is 50, key is test_row_0/C:col10/1732742174833/Put/seqid=0 2024-11-27T21:16:16,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742368_1544 (size=13221) 2024-11-27T21:16:16,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/30864e07b47341e893796e4ab664df12 2024-11-27T21:16:16,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/d1659b4bfd6e4892879866e56c860eaf is 50, key is test_row_0/B:col10/1732742174980/Put/seqid=0 2024-11-27T21:16:16,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742369_1545 (size=12301) 2024-11-27T21:16:16,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:16,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:16,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:16,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:16,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:16,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:16,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:16,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T21:16:16,932 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/656c4f37d6b04f9d9bc85429947f7e15 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/656c4f37d6b04f9d9bc85429947f7e15 2024-11-27T21:16:16,935 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf3e016fc40bf5fab56083bda0e72b03/C of cf3e016fc40bf5fab56083bda0e72b03 into 656c4f37d6b04f9d9bc85429947f7e15(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:16,935 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:16,935 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., storeName=cf3e016fc40bf5fab56083bda0e72b03/C, priority=13, startTime=1732742176091; duration=0sec 2024-11-27T21:16:16,935 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:16,935 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf3e016fc40bf5fab56083bda0e72b03:C 2024-11-27T21:16:16,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/d1659b4bfd6e4892879866e56c860eaf 2024-11-27T21:16:16,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/c239c1e305684171b1bf26f6d81ff454 is 50, key is test_row_0/C:col10/1732742174980/Put/seqid=0 2024-11-27T21:16:16,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742370_1546 (size=12301) 2024-11-27T21:16:17,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:17,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:17,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:17,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:17,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:17,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:17,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:17,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:17,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:17,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:17,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/c239c1e305684171b1bf26f6d81ff454 2024-11-27T21:16:17,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/30864e07b47341e893796e4ab664df12 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/30864e07b47341e893796e4ab664df12 2024-11-27T21:16:17,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/30864e07b47341e893796e4ab664df12, entries=150, sequenceid=498, filesize=12.0 K 2024-11-27T21:16:17,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/d1659b4bfd6e4892879866e56c860eaf as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d1659b4bfd6e4892879866e56c860eaf 2024-11-27T21:16:17,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:17,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:17,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. as already flushing 2024-11-27T21:16:17,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:17,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:17,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:17,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d1659b4bfd6e4892879866e56c860eaf, entries=150, sequenceid=498, filesize=12.0 K 2024-11-27T21:16:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:17,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/c239c1e305684171b1bf26f6d81ff454 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/c239c1e305684171b1bf26f6d81ff454 2024-11-27T21:16:17,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/c239c1e305684171b1bf26f6d81ff454, entries=150, sequenceid=498, filesize=12.0 K 2024-11-27T21:16:17,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=6.71 KB/6870 for cf3e016fc40bf5fab56083bda0e72b03 in 1240ms, sequenceid=498, compaction requested=false 2024-11-27T21:16:17,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:17,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T21:16:17,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:17,532 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-27T21:16:17,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:17,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:17,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:17,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:17,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:17,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:17,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/9e63aa5397dc426d91ef213c153a2fb0 is 50, key is test_row_1/A:col10/1732742176159/Put/seqid=0 2024-11-27T21:16:17,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742371_1547 (size=7415) 2024-11-27T21:16:17,939 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/9e63aa5397dc426d91ef213c153a2fb0 2024-11-27T21:16:17,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/1e3d7baaf1cf43fcb411de0d9f55e01d is 50, key is test_row_1/B:col10/1732742176159/Put/seqid=0 2024-11-27T21:16:17,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742372_1548 (size=7415) 2024-11-27T21:16:18,347 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/1e3d7baaf1cf43fcb411de0d9f55e01d 2024-11-27T21:16:18,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/ca9ace87b2dc4c479b1def1794be9068 is 50, key is test_row_1/C:col10/1732742176159/Put/seqid=0 2024-11-27T21:16:18,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742373_1549 (size=7415) 2024-11-27T21:16:18,756 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/ca9ace87b2dc4c479b1def1794be9068 2024-11-27T21:16:18,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/9e63aa5397dc426d91ef213c153a2fb0 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9e63aa5397dc426d91ef213c153a2fb0 2024-11-27T21:16:18,762 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9e63aa5397dc426d91ef213c153a2fb0, entries=50, sequenceid=505, filesize=7.2 K 2024-11-27T21:16:18,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/1e3d7baaf1cf43fcb411de0d9f55e01d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/1e3d7baaf1cf43fcb411de0d9f55e01d 2024-11-27T21:16:18,766 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/1e3d7baaf1cf43fcb411de0d9f55e01d, entries=50, sequenceid=505, filesize=7.2 K 2024-11-27T21:16:18,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/ca9ace87b2dc4c479b1def1794be9068 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/ca9ace87b2dc4c479b1def1794be9068 2024-11-27T21:16:18,768 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/ca9ace87b2dc4c479b1def1794be9068, entries=50, sequenceid=505, filesize=7.2 K 2024-11-27T21:16:18,769 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for cf3e016fc40bf5fab56083bda0e72b03 in 1237ms, sequenceid=505, compaction requested=true 2024-11-27T21:16:18,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for cf3e016fc40bf5fab56083bda0e72b03: 2024-11-27T21:16:18,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:18,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-27T21:16:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-27T21:16:18,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-27T21:16:18,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9880 sec 2024-11-27T21:16:18,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 3.9920 sec 2024-11-27T21:16:18,870 INFO [Thread-2061 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2061. 2024-11-27T21:16:18,870 DEBUG [Thread-2061 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:18,870 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:18,870 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:18,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T21:16:18,949 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1013 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3034 rows 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1004 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3012 rows 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1010 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3025 rows 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1008 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3024 rows 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1006 2024-11-27T21:16:18,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3014 rows 2024-11-27T21:16:18,949 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T21:16:18,949 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:16:18,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:18,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:18,950 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:18,950 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T21:16:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T21:16:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T21:16:18,953 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742178953"}]},"ts":"1732742178953"} 2024-11-27T21:16:18,954 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T21:16:18,954 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T21:16:18,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:16:18,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf3e016fc40bf5fab56083bda0e72b03, UNASSIGN}] 2024-11-27T21:16:18,956 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf3e016fc40bf5fab56083bda0e72b03, UNASSIGN 2024-11-27T21:16:18,957 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=cf3e016fc40bf5fab56083bda0e72b03, regionState=CLOSING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:16:18,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf3e016fc40bf5fab56083bda0e72b03, UNASSIGN because future has completed 2024-11-27T21:16:18,958 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:16:18,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:16:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T21:16:19,110 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:19,110 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:16:19,110 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing cf3e016fc40bf5fab56083bda0e72b03, disabling compactions & flushes 2024-11-27T21:16:19,110 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:19,110 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:19,110 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. after waiting 0 ms 2024-11-27T21:16:19,110 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:19,110 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(2902): Flushing cf3e016fc40bf5fab56083bda0e72b03 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-27T21:16:19,110 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=A 2024-11-27T21:16:19,110 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:19,110 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=B 2024-11-27T21:16:19,110 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:19,110 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf3e016fc40bf5fab56083bda0e72b03, store=C 2024-11-27T21:16:19,110 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:19,113 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/da149b44681b43dcaf110d1996faf097 is 50, key is test_row_2/A:col10/1732742178869/Put/seqid=0 2024-11-27T21:16:19,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742374_1550 (size=7415) 2024-11-27T21:16:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T21:16:19,517 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/da149b44681b43dcaf110d1996faf097 2024-11-27T21:16:19,522 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/c71e986248294f1cb688af3024a16536 is 50, key is test_row_2/B:col10/1732742178869/Put/seqid=0 2024-11-27T21:16:19,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742375_1551 (size=7415) 2024-11-27T21:16:19,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T21:16:19,925 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/c71e986248294f1cb688af3024a16536 2024-11-27T21:16:19,930 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/d782a29b51434af6b0208ede65896ddf is 50, key is test_row_2/C:col10/1732742178869/Put/seqid=0 2024-11-27T21:16:19,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742376_1552 (size=7415) 2024-11-27T21:16:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T21:16:20,333 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/d782a29b51434af6b0208ede65896ddf 2024-11-27T21:16:20,336 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/A/da149b44681b43dcaf110d1996faf097 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/da149b44681b43dcaf110d1996faf097 2024-11-27T21:16:20,339 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/da149b44681b43dcaf110d1996faf097, entries=50, sequenceid=509, filesize=7.2 K 2024-11-27T21:16:20,339 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/B/c71e986248294f1cb688af3024a16536 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/c71e986248294f1cb688af3024a16536 2024-11-27T21:16:20,341 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/c71e986248294f1cb688af3024a16536, entries=50, sequenceid=509, filesize=7.2 K 2024-11-27T21:16:20,342 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/.tmp/C/d782a29b51434af6b0208ede65896ddf as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/d782a29b51434af6b0208ede65896ddf 2024-11-27T21:16:20,344 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/d782a29b51434af6b0208ede65896ddf, entries=50, sequenceid=509, filesize=7.2 K 2024-11-27T21:16:20,345 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for cf3e016fc40bf5fab56083bda0e72b03 in 1235ms, sequenceid=509, compaction requested=true 2024-11-27T21:16:20,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9836a419f0754acebebb22b99f06a1bc, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/5fdb9746d79643a884c5b1e9148d01c4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/0423ade818da4534ac2fd089c0619bab, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/6654853c912e4ae99cd5c0141e346587, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/a880a5b6ac6f4a76a555b281ee62a5e6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/5127283fcac34be99fff394ec15142f2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/f24f0c9585ab41bc9d981f0d50b9b27f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/fe0903bf936f4db9b1f2678028ae42f0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/c18d76137a974c2fb05a0638fac31d8b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/2443311d0db24dda9b8cbbb322a7bfe0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/65c5aa4063f94b4783c5f577223f0d5e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/206c47c4ff9146f88327dbab6a361f77, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/e9ad557cb5244e999a1f699674cb4a40, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/bd5f26c9478a49cab2ad713530e55ac2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/1456e010e1be4eb5987317e02be86349, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/087be4b323ff41609876eb29e216ce80, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/38fafd8f42a54d0984cb2392250cac4c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/c11d40c95895430ab0a884a44ffad0b5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9c2692e43aae461da81aa03f38c1fe2a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/47f1bd0399db46b5b17712499834a502, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/4bd297486aad4ffdaef7451c8d5b3ce1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9fe529a4aea64dbbac5060e27e15c9b9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/da93eefc142342209b76739b9a28dc63, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/b3ab7979957f4bd58539d7a933d7fd6e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/6a6c7640a53e40a5a4f5af65209cd38b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/a4c9f8aff66b4f8c82221b7370ced140, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/94f707994deb4502a00d7399519da55b] to archive 2024-11-27T21:16:20,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:16:20,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9836a419f0754acebebb22b99f06a1bc to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9836a419f0754acebebb22b99f06a1bc 2024-11-27T21:16:20,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/5fdb9746d79643a884c5b1e9148d01c4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/5fdb9746d79643a884c5b1e9148d01c4 2024-11-27T21:16:20,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/0423ade818da4534ac2fd089c0619bab to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/0423ade818da4534ac2fd089c0619bab 2024-11-27T21:16:20,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/6654853c912e4ae99cd5c0141e346587 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/6654853c912e4ae99cd5c0141e346587 2024-11-27T21:16:20,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/a880a5b6ac6f4a76a555b281ee62a5e6 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/a880a5b6ac6f4a76a555b281ee62a5e6 2024-11-27T21:16:20,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/5127283fcac34be99fff394ec15142f2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/5127283fcac34be99fff394ec15142f2 2024-11-27T21:16:20,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/f24f0c9585ab41bc9d981f0d50b9b27f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/f24f0c9585ab41bc9d981f0d50b9b27f 2024-11-27T21:16:20,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/fe0903bf936f4db9b1f2678028ae42f0 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/fe0903bf936f4db9b1f2678028ae42f0 2024-11-27T21:16:20,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/c18d76137a974c2fb05a0638fac31d8b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/c18d76137a974c2fb05a0638fac31d8b 2024-11-27T21:16:20,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/2443311d0db24dda9b8cbbb322a7bfe0 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/2443311d0db24dda9b8cbbb322a7bfe0 2024-11-27T21:16:20,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/65c5aa4063f94b4783c5f577223f0d5e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/65c5aa4063f94b4783c5f577223f0d5e 2024-11-27T21:16:20,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/206c47c4ff9146f88327dbab6a361f77 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/206c47c4ff9146f88327dbab6a361f77 2024-11-27T21:16:20,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/e9ad557cb5244e999a1f699674cb4a40 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/e9ad557cb5244e999a1f699674cb4a40 2024-11-27T21:16:20,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/bd5f26c9478a49cab2ad713530e55ac2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/bd5f26c9478a49cab2ad713530e55ac2 2024-11-27T21:16:20,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/1456e010e1be4eb5987317e02be86349 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/1456e010e1be4eb5987317e02be86349 2024-11-27T21:16:20,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/087be4b323ff41609876eb29e216ce80 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/087be4b323ff41609876eb29e216ce80 2024-11-27T21:16:20,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/38fafd8f42a54d0984cb2392250cac4c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/38fafd8f42a54d0984cb2392250cac4c 2024-11-27T21:16:20,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/c11d40c95895430ab0a884a44ffad0b5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/c11d40c95895430ab0a884a44ffad0b5 2024-11-27T21:16:20,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9c2692e43aae461da81aa03f38c1fe2a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9c2692e43aae461da81aa03f38c1fe2a 2024-11-27T21:16:20,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/47f1bd0399db46b5b17712499834a502 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/47f1bd0399db46b5b17712499834a502 2024-11-27T21:16:20,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/4bd297486aad4ffdaef7451c8d5b3ce1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/4bd297486aad4ffdaef7451c8d5b3ce1 2024-11-27T21:16:20,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9fe529a4aea64dbbac5060e27e15c9b9 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9fe529a4aea64dbbac5060e27e15c9b9 2024-11-27T21:16:20,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/da93eefc142342209b76739b9a28dc63 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/da93eefc142342209b76739b9a28dc63 2024-11-27T21:16:20,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/b3ab7979957f4bd58539d7a933d7fd6e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/b3ab7979957f4bd58539d7a933d7fd6e 2024-11-27T21:16:20,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/6a6c7640a53e40a5a4f5af65209cd38b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/6a6c7640a53e40a5a4f5af65209cd38b 2024-11-27T21:16:20,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/a4c9f8aff66b4f8c82221b7370ced140 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/a4c9f8aff66b4f8c82221b7370ced140 2024-11-27T21:16:20,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/94f707994deb4502a00d7399519da55b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/94f707994deb4502a00d7399519da55b 2024-11-27T21:16:20,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d33cf303383445ed9341b6bc7f740f35, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/21bc047e98dc493e8c57f7d75fb5bfe0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6f66d12973f44a95a9184af7f3adea48, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/a90081ab6a6047228cae4567708e58ea, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/0f8f74fe23e848b2938a8e593cb2bc69, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8caa4e83078d459c882bce3c95abc8c7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/5ab27b9e698d462a90c5a8ccb0622905, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/86285e2662824bd1aa33a14054a798c6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/1579f6ca0e2645b49a76f7dd6583ec2d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/7c1a44383a7a4ee7975df8e61fccb666, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d83636f25a364144a9ec4eb2973af953, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/66ba9f28faa442f9aca8bbe895c217bd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/077b9ce8f37647b2953aeb812c260b4c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6e9a6112e0e24284937cecb50d922d12, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/3ecdf921f2eb44978f8069aaa63cf3dc, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8e7fe7cc7b3b4fe0a219d3087ef7c346, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/81fc25db0f0f45129ec3c90f96f80c7d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/f518f934cd84487aaa7f5f5c0d11052a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8a3268c93fb4451794aac5fee15777c4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/879ace997e7e4380b8dc17aa7d3c2206, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/b8ceb1d570fb46a2811d3c5fbcb1151b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/3a4afabceb2f49ff82911ee977d54299, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/077e03dea9db450d857c99d0a6bbd3f1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/7ee51a22b0c84c83a432a3ce5f0e6c8e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d805d205d8574c58a586dcf8e4e05385, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6e344eac0f914ce2a496d3b81fde118d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/c1040852b5924c05ac972e294a7249d0] to archive 2024-11-27T21:16:20,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:16:20,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d33cf303383445ed9341b6bc7f740f35 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d33cf303383445ed9341b6bc7f740f35 2024-11-27T21:16:20,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/21bc047e98dc493e8c57f7d75fb5bfe0 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/21bc047e98dc493e8c57f7d75fb5bfe0 2024-11-27T21:16:20,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6f66d12973f44a95a9184af7f3adea48 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6f66d12973f44a95a9184af7f3adea48 2024-11-27T21:16:20,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/a90081ab6a6047228cae4567708e58ea to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/a90081ab6a6047228cae4567708e58ea 2024-11-27T21:16:20,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/0f8f74fe23e848b2938a8e593cb2bc69 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/0f8f74fe23e848b2938a8e593cb2bc69 2024-11-27T21:16:20,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8caa4e83078d459c882bce3c95abc8c7 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8caa4e83078d459c882bce3c95abc8c7 2024-11-27T21:16:20,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/5ab27b9e698d462a90c5a8ccb0622905 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/5ab27b9e698d462a90c5a8ccb0622905 2024-11-27T21:16:20,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/86285e2662824bd1aa33a14054a798c6 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/86285e2662824bd1aa33a14054a798c6 2024-11-27T21:16:20,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/1579f6ca0e2645b49a76f7dd6583ec2d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/1579f6ca0e2645b49a76f7dd6583ec2d 2024-11-27T21:16:20,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/7c1a44383a7a4ee7975df8e61fccb666 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/7c1a44383a7a4ee7975df8e61fccb666 2024-11-27T21:16:20,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d83636f25a364144a9ec4eb2973af953 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d83636f25a364144a9ec4eb2973af953 2024-11-27T21:16:20,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/66ba9f28faa442f9aca8bbe895c217bd to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/66ba9f28faa442f9aca8bbe895c217bd 2024-11-27T21:16:20,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/077b9ce8f37647b2953aeb812c260b4c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/077b9ce8f37647b2953aeb812c260b4c 2024-11-27T21:16:20,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6e9a6112e0e24284937cecb50d922d12 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6e9a6112e0e24284937cecb50d922d12 2024-11-27T21:16:20,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/3ecdf921f2eb44978f8069aaa63cf3dc to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/3ecdf921f2eb44978f8069aaa63cf3dc 2024-11-27T21:16:20,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8e7fe7cc7b3b4fe0a219d3087ef7c346 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8e7fe7cc7b3b4fe0a219d3087ef7c346 2024-11-27T21:16:20,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/81fc25db0f0f45129ec3c90f96f80c7d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/81fc25db0f0f45129ec3c90f96f80c7d 2024-11-27T21:16:20,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/f518f934cd84487aaa7f5f5c0d11052a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/f518f934cd84487aaa7f5f5c0d11052a 2024-11-27T21:16:20,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8a3268c93fb4451794aac5fee15777c4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/8a3268c93fb4451794aac5fee15777c4 2024-11-27T21:16:20,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/879ace997e7e4380b8dc17aa7d3c2206 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/879ace997e7e4380b8dc17aa7d3c2206 2024-11-27T21:16:20,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/b8ceb1d570fb46a2811d3c5fbcb1151b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/b8ceb1d570fb46a2811d3c5fbcb1151b 2024-11-27T21:16:20,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/3a4afabceb2f49ff82911ee977d54299 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/3a4afabceb2f49ff82911ee977d54299 2024-11-27T21:16:20,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/077e03dea9db450d857c99d0a6bbd3f1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/077e03dea9db450d857c99d0a6bbd3f1 2024-11-27T21:16:20,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/7ee51a22b0c84c83a432a3ce5f0e6c8e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/7ee51a22b0c84c83a432a3ce5f0e6c8e 2024-11-27T21:16:20,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d805d205d8574c58a586dcf8e4e05385 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d805d205d8574c58a586dcf8e4e05385 2024-11-27T21:16:20,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6e344eac0f914ce2a496d3b81fde118d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/6e344eac0f914ce2a496d3b81fde118d 2024-11-27T21:16:20,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/c1040852b5924c05ac972e294a7249d0 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/c1040852b5924c05ac972e294a7249d0 2024-11-27T21:16:20,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/4d982433417f408ba3e5ef34a22520c5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/07da68eac79b448e9e2d1b7ba4970bd4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/3d383d1d29054eec87e36288ff4e7a82, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a6f2587577194e24abd8bc9ddf38fd98, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/ca8a935beca746169ef7a48bcc11c60e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/8949b9b71e824fbe92ded10c9d3fc3e3, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/6bcfc11665464d04a33f9900727d3bce, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9e122496683048e38e8dddb6aeb4a061, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/5397eebf1a534d91bff2feaca0297c44, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/fd5898fd222145df9e8336f22bb2d905, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/0671fb0e7cf7460092f5c7449de4c4bd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/840f5cda68874cc5acdc4579f6b99a37, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9b4e5e97ecad437bace500f492c7bb56, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/bcb71a7b770741b0bd247779c7379ee6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/6e17242580944a2b95e4193b829c1d97, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/dec703e6b70e4582bc99ad7cbe2e5a7f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a98765a9276140579ecae6d79c5fef3e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/7d51487c6ac346e9ba7b3cca86abf956, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/52cbfab76b0749bd972d147c5a47f62a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/64cbee97c7bc4e1988b80abe86b5d2f2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9bb8d1456c0e4a89bb5983d32fedf257, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/0b7709f09c8b498c9f3d7756c12d5efe, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/d41fa4fd1d8841f6a7801c5cf37e9566, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/1b83051fe39b453eb8e5da354abae6d3, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/09346b9085604b16a4c0ab6c43adac1a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/10720badecfc49c9a2dcf4dd88d0d263, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a52c9fc73bb5458e97cc909247d13f64] to archive 2024-11-27T21:16:20,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:16:20,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/4d982433417f408ba3e5ef34a22520c5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/4d982433417f408ba3e5ef34a22520c5 2024-11-27T21:16:20,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/07da68eac79b448e9e2d1b7ba4970bd4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/07da68eac79b448e9e2d1b7ba4970bd4 2024-11-27T21:16:20,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/3d383d1d29054eec87e36288ff4e7a82 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/3d383d1d29054eec87e36288ff4e7a82 2024-11-27T21:16:20,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a6f2587577194e24abd8bc9ddf38fd98 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a6f2587577194e24abd8bc9ddf38fd98 2024-11-27T21:16:20,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/ca8a935beca746169ef7a48bcc11c60e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/ca8a935beca746169ef7a48bcc11c60e 2024-11-27T21:16:20,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/8949b9b71e824fbe92ded10c9d3fc3e3 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/8949b9b71e824fbe92ded10c9d3fc3e3 2024-11-27T21:16:20,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/6bcfc11665464d04a33f9900727d3bce to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/6bcfc11665464d04a33f9900727d3bce 2024-11-27T21:16:20,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9e122496683048e38e8dddb6aeb4a061 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9e122496683048e38e8dddb6aeb4a061 2024-11-27T21:16:20,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/5397eebf1a534d91bff2feaca0297c44 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/5397eebf1a534d91bff2feaca0297c44 2024-11-27T21:16:20,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/fd5898fd222145df9e8336f22bb2d905 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/fd5898fd222145df9e8336f22bb2d905 2024-11-27T21:16:20,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/0671fb0e7cf7460092f5c7449de4c4bd to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/0671fb0e7cf7460092f5c7449de4c4bd 2024-11-27T21:16:20,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/840f5cda68874cc5acdc4579f6b99a37 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/840f5cda68874cc5acdc4579f6b99a37 2024-11-27T21:16:20,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9b4e5e97ecad437bace500f492c7bb56 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9b4e5e97ecad437bace500f492c7bb56 2024-11-27T21:16:20,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/bcb71a7b770741b0bd247779c7379ee6 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/bcb71a7b770741b0bd247779c7379ee6 2024-11-27T21:16:20,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/6e17242580944a2b95e4193b829c1d97 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/6e17242580944a2b95e4193b829c1d97 2024-11-27T21:16:20,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/dec703e6b70e4582bc99ad7cbe2e5a7f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/dec703e6b70e4582bc99ad7cbe2e5a7f 2024-11-27T21:16:20,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a98765a9276140579ecae6d79c5fef3e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a98765a9276140579ecae6d79c5fef3e 2024-11-27T21:16:20,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/7d51487c6ac346e9ba7b3cca86abf956 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/7d51487c6ac346e9ba7b3cca86abf956 2024-11-27T21:16:20,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/52cbfab76b0749bd972d147c5a47f62a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/52cbfab76b0749bd972d147c5a47f62a 2024-11-27T21:16:20,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/64cbee97c7bc4e1988b80abe86b5d2f2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/64cbee97c7bc4e1988b80abe86b5d2f2 2024-11-27T21:16:20,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9bb8d1456c0e4a89bb5983d32fedf257 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/9bb8d1456c0e4a89bb5983d32fedf257 2024-11-27T21:16:20,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/0b7709f09c8b498c9f3d7756c12d5efe to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/0b7709f09c8b498c9f3d7756c12d5efe 2024-11-27T21:16:20,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/d41fa4fd1d8841f6a7801c5cf37e9566 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/d41fa4fd1d8841f6a7801c5cf37e9566 2024-11-27T21:16:20,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/1b83051fe39b453eb8e5da354abae6d3 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/1b83051fe39b453eb8e5da354abae6d3 2024-11-27T21:16:20,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/09346b9085604b16a4c0ab6c43adac1a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/09346b9085604b16a4c0ab6c43adac1a 2024-11-27T21:16:20,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/10720badecfc49c9a2dcf4dd88d0d263 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/10720badecfc49c9a2dcf4dd88d0d263 2024-11-27T21:16:20,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a52c9fc73bb5458e97cc909247d13f64 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/a52c9fc73bb5458e97cc909247d13f64 2024-11-27T21:16:20,413 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/recovered.edits/512.seqid, newMaxSeqId=512, maxSeqId=1 2024-11-27T21:16:20,413 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. 2024-11-27T21:16:20,413 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for cf3e016fc40bf5fab56083bda0e72b03: Waiting for close lock at 1732742179110Running coprocessor pre-close hooks at 1732742179110Disabling compacts and flushes for region at 1732742179110Disabling writes for close at 1732742179110Obtaining lock to block concurrent updates at 1732742179110Preparing flush snapshotting stores in cf3e016fc40bf5fab56083bda0e72b03 at 1732742179110Finished memstore snapshotting TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1732742179111 (+1 ms)Flushing stores of TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03. at 1732742179111Flushing cf3e016fc40bf5fab56083bda0e72b03/A: creating writer at 1732742179111Flushing cf3e016fc40bf5fab56083bda0e72b03/A: appending metadata at 1732742179113 (+2 ms)Flushing cf3e016fc40bf5fab56083bda0e72b03/A: closing flushed file at 1732742179113Flushing cf3e016fc40bf5fab56083bda0e72b03/B: creating writer at 1732742179519 (+406 ms)Flushing cf3e016fc40bf5fab56083bda0e72b03/B: appending metadata at 1732742179521 (+2 ms)Flushing cf3e016fc40bf5fab56083bda0e72b03/B: closing flushed file at 1732742179521Flushing cf3e016fc40bf5fab56083bda0e72b03/C: creating writer at 1732742179928 (+407 ms)Flushing cf3e016fc40bf5fab56083bda0e72b03/C: appending metadata at 1732742179929 (+1 ms)Flushing cf3e016fc40bf5fab56083bda0e72b03/C: closing flushed file at 1732742179929Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6672d53f: reopening flushed file at 1732742180336 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15829439: reopening flushed file at 1732742180339 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e5fe82: reopening flushed file at 1732742180341 (+2 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for cf3e016fc40bf5fab56083bda0e72b03 in 1235ms, sequenceid=509, compaction requested=true at 1732742180345 (+4 ms)Writing region close event to WAL at 1732742180411 (+66 ms)Running coprocessor post-close hooks at 1732742180413 (+2 ms)Closed at 1732742180413 2024-11-27T21:16:20,415 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:20,415 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=cf3e016fc40bf5fab56083bda0e72b03, regionState=CLOSED 2024-11-27T21:16:20,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:16:20,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-27T21:16:20,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure cf3e016fc40bf5fab56083bda0e72b03, server=4585c39bed03,37381,1732742022265 in 1.4590 sec 2024-11-27T21:16:20,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-27T21:16:20,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf3e016fc40bf5fab56083bda0e72b03, UNASSIGN in 1.4620 sec 2024-11-27T21:16:20,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-27T21:16:20,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4650 sec 2024-11-27T21:16:20,422 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742180422"}]},"ts":"1732742180422"} 2024-11-27T21:16:20,423 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T21:16:20,423 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T21:16:20,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4740 sec 2024-11-27T21:16:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T21:16:21,099 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:21,100 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T21:16:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:16:21,101 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:16:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-27T21:16:21,102 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=147, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:16:21,104 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:21,106 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/recovered.edits] 2024-11-27T21:16:21,108 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/30864e07b47341e893796e4ab664df12 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/30864e07b47341e893796e4ab664df12 2024-11-27T21:16:21,109 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/992a313d18ab45b8816c13ecd76b5a74 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/992a313d18ab45b8816c13ecd76b5a74 2024-11-27T21:16:21,109 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9e63aa5397dc426d91ef213c153a2fb0 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/9e63aa5397dc426d91ef213c153a2fb0 2024-11-27T21:16:21,110 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/da149b44681b43dcaf110d1996faf097 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/A/da149b44681b43dcaf110d1996faf097 2024-11-27T21:16:21,112 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/1e3d7baaf1cf43fcb411de0d9f55e01d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/1e3d7baaf1cf43fcb411de0d9f55e01d 2024-11-27T21:16:21,113 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/4a5a923eb01745739950a09e3b6917cf to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/4a5a923eb01745739950a09e3b6917cf 2024-11-27T21:16:21,114 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/c71e986248294f1cb688af3024a16536 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/c71e986248294f1cb688af3024a16536 2024-11-27T21:16:21,114 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d1659b4bfd6e4892879866e56c860eaf to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/B/d1659b4bfd6e4892879866e56c860eaf 2024-11-27T21:16:21,116 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/656c4f37d6b04f9d9bc85429947f7e15 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/656c4f37d6b04f9d9bc85429947f7e15 2024-11-27T21:16:21,117 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/c239c1e305684171b1bf26f6d81ff454 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/c239c1e305684171b1bf26f6d81ff454 2024-11-27T21:16:21,117 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/ca9ace87b2dc4c479b1def1794be9068 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/ca9ace87b2dc4c479b1def1794be9068 2024-11-27T21:16:21,118 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/d782a29b51434af6b0208ede65896ddf to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/C/d782a29b51434af6b0208ede65896ddf 2024-11-27T21:16:21,120 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/recovered.edits/512.seqid to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03/recovered.edits/512.seqid 2024-11-27T21:16:21,120 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/cf3e016fc40bf5fab56083bda0e72b03 2024-11-27T21:16:21,120 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T21:16:21,122 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=147, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:16:21,124 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T21:16:21,125 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T21:16:21,126 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=147, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:16:21,126 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T21:16:21,126 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732742181126"}]},"ts":"9223372036854775807"} 2024-11-27T21:16:21,128 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T21:16:21,128 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cf3e016fc40bf5fab56083bda0e72b03, NAME => 'TestAcidGuarantees,,1732742153888.cf3e016fc40bf5fab56083bda0e72b03.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T21:16:21,128 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T21:16:21,128 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732742181128"}]},"ts":"9223372036854775807"} 2024-11-27T21:16:21,129 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T21:16:21,130 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=147, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:16:21,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-27T21:16:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-27T21:16:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T21:16:21,213 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:21,220 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251 (was 249) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=346 (was 328) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7933 (was 7975) 2024-11-27T21:16:21,228 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=346, ProcessCount=11, AvailableMemoryMB=7932 2024-11-27T21:16:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:16:21,229 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:16:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T21:16:21,231 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T21:16:21,231 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:21,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 148 2024-11-27T21:16:21,232 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T21:16:21,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T21:16:21,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742377_1553 (size=1061) 2024-11-27T21:16:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T21:16:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T21:16:21,639 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de 2024-11-27T21:16:21,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742378_1554 (size=53) 2024-11-27T21:16:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T21:16:21,940 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T21:16:22,044 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:16:22,044 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b6db78cfa2317a7fa812b5763434be10, disabling compactions & flushes 2024-11-27T21:16:22,045 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:22,045 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:22,045 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. after waiting 0 ms 2024-11-27T21:16:22,045 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:22,045 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:22,045 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b6db78cfa2317a7fa812b5763434be10: Waiting for close lock at 1732742182044Disabling compacts and flushes for region at 1732742182044Disabling writes for close at 1732742182045 (+1 ms)Writing region close event to WAL at 1732742182045Closed at 1732742182045 2024-11-27T21:16:22,046 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T21:16:22,046 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732742182046"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732742182046"}]},"ts":"1732742182046"} 2024-11-27T21:16:22,047 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T21:16:22,048 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T21:16:22,048 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742182048"}]},"ts":"1732742182048"} 2024-11-27T21:16:22,049 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T21:16:22,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6db78cfa2317a7fa812b5763434be10, ASSIGN}] 2024-11-27T21:16:22,050 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6db78cfa2317a7fa812b5763434be10, ASSIGN 2024-11-27T21:16:22,051 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6db78cfa2317a7fa812b5763434be10, ASSIGN; state=OFFLINE, location=4585c39bed03,37381,1732742022265; forceNewPlan=false, retain=false 2024-11-27T21:16:22,201 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=b6db78cfa2317a7fa812b5763434be10, regionState=OPENING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:16:22,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6db78cfa2317a7fa812b5763434be10, ASSIGN because future has completed 2024-11-27T21:16:22,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:16:22,357 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:22,358 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7752): Opening region: {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:16:22,358 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:22,358 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:16:22,358 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7794): checking encryption for b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:22,358 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7797): checking classloading for b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:22,359 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:22,360 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:16:22,361 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6db78cfa2317a7fa812b5763434be10 columnFamilyName A 2024-11-27T21:16:22,361 DEBUG [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:22,361 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.HStore(327): Store=b6db78cfa2317a7fa812b5763434be10/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:16:22,361 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:22,362 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:16:22,362 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6db78cfa2317a7fa812b5763434be10 columnFamilyName B 2024-11-27T21:16:22,362 DEBUG [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:22,362 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.HStore(327): Store=b6db78cfa2317a7fa812b5763434be10/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:16:22,362 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:22,363 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:16:22,363 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6db78cfa2317a7fa812b5763434be10 columnFamilyName C 2024-11-27T21:16:22,363 DEBUG [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:22,363 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.HStore(327): Store=b6db78cfa2317a7fa812b5763434be10/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:16:22,364 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:22,364 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1038): replaying wal for b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:22,364 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:22,365 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:22,365 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1048): stopping wal replay for b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:22,365 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1060): Cleaning up temporary data for b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:22,366 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:16:22,367 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1093): writing seq id for b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:22,368 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T21:16:22,369 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1114): Opened b6db78cfa2317a7fa812b5763434be10; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71290990, jitterRate=0.06231853365898132}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:16:22,369 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T21:16:22,370 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1006): Region open journal for b6db78cfa2317a7fa812b5763434be10: Running coprocessor pre-open hook at 1732742182358Writing region info on filesystem at 1732742182358Initializing all the Stores at 1732742182359 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742182359Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742182359Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742182359Cleaning up temporary data from old regions at 1732742182365 (+6 ms)Running coprocessor post-open hooks at 1732742182369 (+4 ms)Region opened successfully at 1732742182370 (+1 ms) 2024-11-27T21:16:22,370 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., pid=150, masterSystemTime=1732742182355 2024-11-27T21:16:22,372 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:22,372 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:22,372 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=b6db78cfa2317a7fa812b5763434be10, regionState=OPEN, openSeqNum=2, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:16:22,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:16:22,375 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-27T21:16:22,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 in 171 msec 2024-11-27T21:16:22,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-27T21:16:22,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6db78cfa2317a7fa812b5763434be10, ASSIGN in 327 msec 2024-11-27T21:16:22,377 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T21:16:22,378 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742182378"}]},"ts":"1732742182378"} 2024-11-27T21:16:22,379 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T21:16:22,379 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T21:16:22,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-27T21:16:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T21:16:23,379 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:23,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185b2823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:23,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:16:23,380 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:16:23,381 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:16:23,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:16:23,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:16:23,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12166527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:23,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:16:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:16:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:23,382 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:16:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4006fb42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:23,383 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:16:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:16:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:16:23,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:16:23,385 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T21:16:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 4585c39bed03,38503,1732742021440 2024-11-27T21:16:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5e76b0fd 2024-11-27T21:16:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T21:16:23,386 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T21:16:23,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T21:16:23,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T21:16:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T21:16:23,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742379_1555 (size=1097) 2024-11-27T21:16:23,799 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-27T21:16:23,799 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-27T21:16:23,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:16:23,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6db78cfa2317a7fa812b5763434be10, REOPEN/MOVE}] 2024-11-27T21:16:23,803 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6db78cfa2317a7fa812b5763434be10, REOPEN/MOVE 2024-11-27T21:16:23,804 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=b6db78cfa2317a7fa812b5763434be10, regionState=CLOSING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:16:23,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6db78cfa2317a7fa812b5763434be10, REOPEN/MOVE because future has completed 2024-11-27T21:16:23,805 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:16:23,805 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:16:23,957 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(122): Close b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:23,957 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:16:23,958 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1722): Closing b6db78cfa2317a7fa812b5763434be10, disabling compactions & flushes 2024-11-27T21:16:23,958 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:23,958 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:23,958 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. after waiting 0 ms 2024-11-27T21:16:23,958 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:23,961 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T21:16:23,962 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:23,962 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1676): Region close journal for b6db78cfa2317a7fa812b5763434be10: Waiting for close lock at 1732742183958Running coprocessor pre-close hooks at 1732742183958Disabling compacts and flushes for region at 1732742183958Disabling writes for close at 1732742183958Writing region close event to WAL at 1732742183958Running coprocessor post-close hooks at 1732742183962 (+4 ms)Closed at 1732742183962 2024-11-27T21:16:23,962 WARN [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegionServer(3299): Not adding moved region record: b6db78cfa2317a7fa812b5763434be10 to self. 2024-11-27T21:16:23,963 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(157): Closed b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:23,964 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=b6db78cfa2317a7fa812b5763434be10, regionState=CLOSED 2024-11-27T21:16:23,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:16:23,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-27T21:16:23,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseRegionProcedure b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 in 160 msec 2024-11-27T21:16:23,967 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6db78cfa2317a7fa812b5763434be10, REOPEN/MOVE; state=CLOSED, location=4585c39bed03,37381,1732742022265; forceNewPlan=false, retain=true 2024-11-27T21:16:24,118 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=b6db78cfa2317a7fa812b5763434be10, regionState=OPENING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6db78cfa2317a7fa812b5763434be10, REOPEN/MOVE because future has completed 2024-11-27T21:16:24,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:16:24,274 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:24,274 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7752): Opening region: {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} 2024-11-27T21:16:24,274 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:24,274 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T21:16:24,275 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7794): checking encryption for b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:24,275 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7797): checking classloading for b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:24,276 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:24,276 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:16:24,277 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6db78cfa2317a7fa812b5763434be10 columnFamilyName A 2024-11-27T21:16:24,277 DEBUG [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:24,278 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.HStore(327): Store=b6db78cfa2317a7fa812b5763434be10/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:16:24,278 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:24,279 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:16:24,279 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6db78cfa2317a7fa812b5763434be10 columnFamilyName B 2024-11-27T21:16:24,279 DEBUG [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:24,279 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.HStore(327): Store=b6db78cfa2317a7fa812b5763434be10/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:16:24,279 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:24,280 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T21:16:24,280 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6db78cfa2317a7fa812b5763434be10 columnFamilyName C 2024-11-27T21:16:24,280 DEBUG [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:24,280 INFO [StoreOpener-b6db78cfa2317a7fa812b5763434be10-1 {}] regionserver.HStore(327): Store=b6db78cfa2317a7fa812b5763434be10/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T21:16:24,280 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:24,281 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1038): replaying wal for b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:24,281 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:24,282 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:24,282 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1048): stopping wal replay for b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:24,282 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1060): Cleaning up temporary data for b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:24,283 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T21:16:24,284 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1093): writing seq id for b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:24,285 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1114): Opened b6db78cfa2317a7fa812b5763434be10; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65893793, jitterRate=-0.018105968832969666}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T21:16:24,285 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:24,285 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1006): Region open journal for b6db78cfa2317a7fa812b5763434be10: Running coprocessor pre-open hook at 1732742184275Writing region info on filesystem at 1732742184275Initializing all the Stores at 1732742184275Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742184275Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742184276 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732742184276Cleaning up temporary data from old regions at 1732742184282 (+6 ms)Running coprocessor post-open hooks at 1732742184285 (+3 ms)Region opened successfully at 1732742184285 2024-11-27T21:16:24,286 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., pid=155, masterSystemTime=1732742184271 2024-11-27T21:16:24,287 DEBUG [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:24,287 INFO [RS_OPEN_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:24,288 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=b6db78cfa2317a7fa812b5763434be10, regionState=OPEN, openSeqNum=5, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:16:24,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=153 2024-11-27T21:16:24,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 in 170 msec 2024-11-27T21:16:24,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-27T21:16:24,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6db78cfa2317a7fa812b5763434be10, REOPEN/MOVE in 487 msec 2024-11-27T21:16:24,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-27T21:16:24,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-27T21:16:24,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 906 msec 2024-11-27T21:16:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-27T21:16:24,296 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:24,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,296 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:16:24,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:16:24,297 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:16:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:16:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:16:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f8e024, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:16:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:16:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:24,298 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:16:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23016d42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,299 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:16:24,299 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:16:24,300 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:16:24,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:16:24,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:16:24,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef14dac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:16:24,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:16:24,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:24,301 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:16:24,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3858cb27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:16:24,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:16:24,302 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:16:24,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:16:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:16:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36d26339, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:16:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:16:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:24,303 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:16:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c17e516, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:16:24,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:16:24,305 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:16:24,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:16:24,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:16:24,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47dd6829, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:16:24,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:16:24,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:24,306 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:16:24,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@325036dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,307 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:16:24,307 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:16:24,307 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:16:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:16:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:16:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3442c1be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:16:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:16:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:24,308 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:16:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56f38127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:16:24,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:16:24,310 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:16:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:16:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:16:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30381f76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:16:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:16:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:24,310 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:16:24,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382ade5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,311 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:16:24,311 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:16:24,312 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:16:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:16:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:16:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c87c422, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:16:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:16:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:24,313 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:16:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@458d0597, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:16:24,314 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:16:24,315 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:16:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:16:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:16:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57d67de6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:16:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:16:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:24,316 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:16:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29907fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:16:24,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:16:24,317 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:16:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:16:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:16:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e8d0fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:16:24,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:16:24,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:24,318 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:16:24,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c24e228, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e766e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,319 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4585c39bed03,38503,-1 for getting cluster id 2024-11-27T21:16:24,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T21:16:24,320 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74733cde-8d64-4f1f-94fd-09392088645c' 2024-11-27T21:16:24,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T21:16:24,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74733cde-8d64-4f1f-94fd-09392088645c" 2024-11-27T21:16:24,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56beed6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4585c39bed03,38503,-1] 2024-11-27T21:16:24,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T21:16:24,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:24,321 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T21:16:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f785924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T21:16:24,322 DEBUG [Thread-2485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:16:24,322 DEBUG [Thread-2487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:16:24,322 DEBUG [Thread-2478 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:16:24,322 DEBUG [Thread-2476 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:16:24,322 DEBUG [Thread-2491 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:16:24,322 DEBUG [Thread-2472 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:16:24,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:16:24,323 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:16:24,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:16:24,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:16:24,323 DEBUG [Thread-2480 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:16:24,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:16:24,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:16:24,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:16:24,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:16:24,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:16:24,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:16:24,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:16:24,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:16:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:16:24,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:16:24,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:16:24,324 DEBUG [Thread-2474 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:16:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:16:24,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:16:24,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:16:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:16:24,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:16:24,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:16:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-27T21:16:24,325 DEBUG [Thread-2483 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:16:24,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:16:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:16:24,326 DEBUG [Thread-2489 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T21:16:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:16:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:16:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T21:16:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:16:24,326 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:16:24,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:16:24,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:16:24,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4585c39bed03,37381,1732742022265, seqNum=-1] 2024-11-27T21:16:24,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:16:24,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T21:16:24,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:16:24,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:16:24,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:16:24,329 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:16:24,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:16:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:16:24,330 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T21:16:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:16:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:16:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:16:24,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:16:24,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5] 2024-11-27T21:16:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:24,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:16:24,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:24,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:24,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:24,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:24,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:24,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:24,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112787bbcc0d99ea4e0ab16d4c3e762cb0f5_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742184332/Put/seqid=0 2024-11-27T21:16:24,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742380_1556 (size=12154) 2024-11-27T21:16:24,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742184379, max timestamp=1732742184418Num uniques 859; 2024-11-27T21:16:24,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:24,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742184379, max timestamp=1732742184418Num uniques 859; 2024-11-27T21:16:24,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742184379, max timestamp=1732742184418Num uniques 859; 2024-11-27T21:16:24,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:24,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T21:16:24,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:24,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742244438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742244439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742244440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:24,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742244446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742244446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:24,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T21:16:24,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:24,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:24,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:24,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:24,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:24,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742244549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742244549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:24,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742244549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:24,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742244559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742244559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:24,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T21:16:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:24,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T21:16:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:24,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742244759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742244759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:24,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742244759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:24,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742244768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:24,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742244768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:24,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:24,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:24,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:24,790 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:24,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T21:16:24,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:24,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:24,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:24,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:24,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:24,796 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112787bbcc0d99ea4e0ab16d4c3e762cb0f5_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112787bbcc0d99ea4e0ab16d4c3e762cb0f5_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:24,797 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/1e3c5263d70748e5a7743eacf3c33994, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:24,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/1e3c5263d70748e5a7743eacf3c33994 is 175, key is test_row_0/A:col10/1732742184332/Put/seqid=0 2024-11-27T21:16:24,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742381_1557 (size=30955) 2024-11-27T21:16:24,802 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/1e3c5263d70748e5a7743eacf3c33994 2024-11-27T21:16:24,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/e5f15a80dfcc4311addc28fee9d1cee4 is 50, key is test_row_0/B:col10/1732742184332/Put/seqid=0 2024-11-27T21:16:24,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742382_1558 (size=12001) 2024-11-27T21:16:24,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/e5f15a80dfcc4311addc28fee9d1cee4 2024-11-27T21:16:24,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/f031a49e63ee447c8b7b09fc763d949a is 50, key is test_row_0/C:col10/1732742184332/Put/seqid=0 2024-11-27T21:16:24,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742383_1559 (size=12001) 2024-11-27T21:16:24,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T21:16:24,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:24,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:24,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:24,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:24,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T21:16:25,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742245069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742245069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742245069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742245079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742245079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T21:16:25,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:25,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:25,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:25,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:25,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:25,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T21:16:25,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:25,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:25,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:25,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:25,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:25,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:25,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/f031a49e63ee447c8b7b09fc763d949a 2024-11-27T21:16:25,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/1e3c5263d70748e5a7743eacf3c33994 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/1e3c5263d70748e5a7743eacf3c33994 2024-11-27T21:16:25,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/1e3c5263d70748e5a7743eacf3c33994, entries=150, sequenceid=14, filesize=30.2 K 2024-11-27T21:16:25,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/e5f15a80dfcc4311addc28fee9d1cee4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/e5f15a80dfcc4311addc28fee9d1cee4 2024-11-27T21:16:25,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/e5f15a80dfcc4311addc28fee9d1cee4, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T21:16:25,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/f031a49e63ee447c8b7b09fc763d949a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/f031a49e63ee447c8b7b09fc763d949a 2024-11-27T21:16:25,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/f031a49e63ee447c8b7b09fc763d949a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T21:16:25,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b6db78cfa2317a7fa812b5763434be10 in 941ms, sequenceid=14, compaction requested=false 2024-11-27T21:16:25,277 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T21:16:25,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:25,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T21:16:25,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:25,404 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-27T21:16:25,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:25,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:25,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:25,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:25,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:25,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:25,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272c06762195134b2fa46538f5597c4b9a_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742184444/Put/seqid=0 2024-11-27T21:16:25,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742384_1560 (size=12154) 2024-11-27T21:16:25,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:25,424 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272c06762195134b2fa46538f5597c4b9a_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272c06762195134b2fa46538f5597c4b9a_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:25,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/e35da90759984e349961dbb671e977a2, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:25,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/e35da90759984e349961dbb671e977a2 is 175, key is test_row_0/A:col10/1732742184444/Put/seqid=0 2024-11-27T21:16:25,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742385_1561 (size=30955) 2024-11-27T21:16:25,432 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/e35da90759984e349961dbb671e977a2 2024-11-27T21:16:25,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/90b90cbc7fd24b78bb2425a7c6761fef is 50, key is test_row_0/B:col10/1732742184444/Put/seqid=0 2024-11-27T21:16:25,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742386_1562 (size=12001) 2024-11-27T21:16:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T21:16:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:25,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:25,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742245594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742245595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742245595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742245596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742245596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742245709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742245709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742245709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742245709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742245710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,843 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/90b90cbc7fd24b78bb2425a7c6761fef 2024-11-27T21:16:25,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/617fe78eb1dc4c17b1afeef9aa90ad9b is 50, key is test_row_0/C:col10/1732742184444/Put/seqid=0 2024-11-27T21:16:25,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742387_1563 (size=12001) 2024-11-27T21:16:25,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742245919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742245919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742245919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742245919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742245920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:25,931 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T21:16:26,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742246229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:26,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742246229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:26,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742246229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:26,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742246229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:26,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742246229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:26,256 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/617fe78eb1dc4c17b1afeef9aa90ad9b 2024-11-27T21:16:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/e35da90759984e349961dbb671e977a2 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/e35da90759984e349961dbb671e977a2 2024-11-27T21:16:26,263 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/e35da90759984e349961dbb671e977a2, entries=150, sequenceid=49, filesize=30.2 K 2024-11-27T21:16:26,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/90b90cbc7fd24b78bb2425a7c6761fef as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/90b90cbc7fd24b78bb2425a7c6761fef 2024-11-27T21:16:26,267 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/90b90cbc7fd24b78bb2425a7c6761fef, entries=150, sequenceid=49, filesize=11.7 K 2024-11-27T21:16:26,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/617fe78eb1dc4c17b1afeef9aa90ad9b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/617fe78eb1dc4c17b1afeef9aa90ad9b 2024-11-27T21:16:26,270 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/617fe78eb1dc4c17b1afeef9aa90ad9b, entries=150, sequenceid=49, filesize=11.7 K 2024-11-27T21:16:26,271 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for b6db78cfa2317a7fa812b5763434be10 in 867ms, sequenceid=49, compaction requested=false 2024-11-27T21:16:26,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:26,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:26,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-27T21:16:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-27T21:16:26,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-27T21:16:26,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9440 sec 2024-11-27T21:16:26,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.9530 sec 2024-11-27T21:16:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T21:16:26,470 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:26,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:16:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-27T21:16:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T21:16:26,472 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:16:26,473 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:16:26,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:16:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T21:16:26,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T21:16:26,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:26,626 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:16:26,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:26,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:26,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:26,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:26,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:26,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:26,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127090b98885918475b8defc0cfa49b7cfb_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742185595/Put/seqid=0 2024-11-27T21:16:26,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742388_1564 (size=12154) 2024-11-27T21:16:26,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:26,643 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127090b98885918475b8defc0cfa49b7cfb_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127090b98885918475b8defc0cfa49b7cfb_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:26,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/b79affaa7ded485c827235234ecf1dba, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:26,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/b79affaa7ded485c827235234ecf1dba is 175, key is test_row_0/A:col10/1732742185595/Put/seqid=0 2024-11-27T21:16:26,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742389_1565 (size=30955) 2024-11-27T21:16:26,648 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/b79affaa7ded485c827235234ecf1dba 2024-11-27T21:16:26,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/462689a9b6194a7a8a4e19f149a037de is 50, key is test_row_0/B:col10/1732742185595/Put/seqid=0 2024-11-27T21:16:26,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742390_1566 (size=12001) 2024-11-27T21:16:26,674 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/462689a9b6194a7a8a4e19f149a037de 2024-11-27T21:16:26,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/2c8fe9aaffb04b0a84b13d0c3259ba8a is 50, key is test_row_0/C:col10/1732742185595/Put/seqid=0 2024-11-27T21:16:26,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742391_1567 (size=12001) 2024-11-27T21:16:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:26,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:26,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742186741, max timestamp=1732742186761Num uniques 859; 2024-11-27T21:16:26,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742186741, max timestamp=1732742186761Num uniques 859; 2024-11-27T21:16:26,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742186741, max timestamp=1732742186761Num uniques 859; 2024-11-27T21:16:26,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:26,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:26,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:26,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742246782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T21:16:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:26,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742246782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742246783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:26,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742246783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:26,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742246786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:26,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:26,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742246898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:26,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742246898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:26,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742246899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742246899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742246899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,084 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/2c8fe9aaffb04b0a84b13d0c3259ba8a 2024-11-27T21:16:27,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/b79affaa7ded485c827235234ecf1dba as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/b79affaa7ded485c827235234ecf1dba 2024-11-27T21:16:27,090 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/b79affaa7ded485c827235234ecf1dba, entries=150, sequenceid=59, filesize=30.2 K 2024-11-27T21:16:27,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/462689a9b6194a7a8a4e19f149a037de as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/462689a9b6194a7a8a4e19f149a037de 2024-11-27T21:16:27,094 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/462689a9b6194a7a8a4e19f149a037de, entries=150, sequenceid=59, filesize=11.7 K 2024-11-27T21:16:27,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/2c8fe9aaffb04b0a84b13d0c3259ba8a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/2c8fe9aaffb04b0a84b13d0c3259ba8a 2024-11-27T21:16:27,097 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/2c8fe9aaffb04b0a84b13d0c3259ba8a, entries=150, sequenceid=59, filesize=11.7 K 2024-11-27T21:16:27,098 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b6db78cfa2317a7fa812b5763434be10 in 473ms, sequenceid=59, compaction requested=true 2024-11-27T21:16:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-27T21:16:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T21:16:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-27T21:16:27,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-27T21:16:27,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 626 msec 2024-11-27T21:16:27,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 632 msec 2024-11-27T21:16:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:27,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-27T21:16:27,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:27,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:27,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:27,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:27,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:27,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:27,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276b01410a681d4055a791e96a62bf0c7a_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742187113/Put/seqid=0 2024-11-27T21:16:27,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742247116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742247116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742247116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742247122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742392_1568 (size=12154) 2024-11-27T21:16:27,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742247123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,131 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,135 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276b01410a681d4055a791e96a62bf0c7a_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276b01410a681d4055a791e96a62bf0c7a_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:27,136 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/7d3a39d6415a44d2a557fc7d3d592514, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:27,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/7d3a39d6415a44d2a557fc7d3d592514 is 175, key is test_row_0/A:col10/1732742187113/Put/seqid=0 2024-11-27T21:16:27,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742393_1569 (size=30955) 2024-11-27T21:16:27,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742247229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742247229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742247229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742247229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742247239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742247439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742247439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742247439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742247440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742247449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,541 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/7d3a39d6415a44d2a557fc7d3d592514 2024-11-27T21:16:27,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/c82c21e1817948a3993f8ca6c4ef9f9c is 50, key is test_row_0/B:col10/1732742187113/Put/seqid=0 2024-11-27T21:16:27,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742394_1570 (size=12001) 2024-11-27T21:16:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T21:16:27,609 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:27,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:16:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-27T21:16:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T21:16:27,611 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:16:27,612 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:16:27,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:16:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T21:16:27,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742247748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742247748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742247749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742247749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742247759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:27,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:27,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:27,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:16:27,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:27,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:27,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:27,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:27,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:27,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:16:27,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:27,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:27,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:27,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:27,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T21:16:27,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/c82c21e1817948a3993f8ca6c4ef9f9c 2024-11-27T21:16:27,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/4c37337cfb8d406c9d18632588227697 is 50, key is test_row_0/C:col10/1732742187113/Put/seqid=0 2024-11-27T21:16:27,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742395_1571 (size=12001) 2024-11-27T21:16:28,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:16:28,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:28,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:28,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:28,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:28,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:28,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:16:28,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:28,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:28,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:28,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:28,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T21:16:28,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742248259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:28,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742248259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:28,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742248259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:28,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742248259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:28,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742248269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:28,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/4c37337cfb8d406c9d18632588227697 2024-11-27T21:16:28,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/7d3a39d6415a44d2a557fc7d3d592514 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/7d3a39d6415a44d2a557fc7d3d592514 2024-11-27T21:16:28,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/7d3a39d6415a44d2a557fc7d3d592514, entries=150, sequenceid=96, filesize=30.2 K 2024-11-27T21:16:28,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/c82c21e1817948a3993f8ca6c4ef9f9c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/c82c21e1817948a3993f8ca6c4ef9f9c 2024-11-27T21:16:28,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/c82c21e1817948a3993f8ca6c4ef9f9c, entries=150, sequenceid=96, filesize=11.7 K 2024-11-27T21:16:28,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/4c37337cfb8d406c9d18632588227697 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/4c37337cfb8d406c9d18632588227697 2024-11-27T21:16:28,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:16:28,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:28,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:28,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:28,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:28,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:28,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/4c37337cfb8d406c9d18632588227697, entries=150, sequenceid=96, filesize=11.7 K 2024-11-27T21:16:28,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for b6db78cfa2317a7fa812b5763434be10 in 1265ms, sequenceid=96, compaction requested=true 2024-11-27T21:16:28,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:28,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:16:28,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:28,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:16:28,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:28,378 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:28,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:16:28,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T21:16:28,379 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123820 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:28,379 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/A is initiating minor compaction (all files) 2024-11-27T21:16:28,379 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/A in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:28,380 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/1e3c5263d70748e5a7743eacf3c33994, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/e35da90759984e349961dbb671e977a2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/b79affaa7ded485c827235234ecf1dba, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/7d3a39d6415a44d2a557fc7d3d592514] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=120.9 K 2024-11-27T21:16:28,380 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:28,380 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/1e3c5263d70748e5a7743eacf3c33994, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/e35da90759984e349961dbb671e977a2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/b79affaa7ded485c827235234ecf1dba, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/7d3a39d6415a44d2a557fc7d3d592514] 2024-11-27T21:16:28,380 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:28,380 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e3c5263d70748e5a7743eacf3c33994, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732742184330 2024-11-27T21:16:28,380 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting e35da90759984e349961dbb671e977a2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732742184438 2024-11-27T21:16:28,381 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting b79affaa7ded485c827235234ecf1dba, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732742185594 2024-11-27T21:16:28,381 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:28,381 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d3a39d6415a44d2a557fc7d3d592514, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732742186781 2024-11-27T21:16:28,381 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/B is initiating minor compaction (all files) 2024-11-27T21:16:28,381 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/B in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:28,381 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/e5f15a80dfcc4311addc28fee9d1cee4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/90b90cbc7fd24b78bb2425a7c6761fef, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/462689a9b6194a7a8a4e19f149a037de, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/c82c21e1817948a3993f8ca6c4ef9f9c] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=46.9 K 2024-11-27T21:16:28,381 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5f15a80dfcc4311addc28fee9d1cee4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732742184330 2024-11-27T21:16:28,382 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90b90cbc7fd24b78bb2425a7c6761fef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732742184438 2024-11-27T21:16:28,382 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 462689a9b6194a7a8a4e19f149a037de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732742185594 2024-11-27T21:16:28,382 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting c82c21e1817948a3993f8ca6c4ef9f9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732742186781 2024-11-27T21:16:28,388 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:28,390 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127178a65b2c5654be2a929cac6c8f46894_b6db78cfa2317a7fa812b5763434be10 store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:28,390 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#B#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:28,391 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/a88d7bb8a60149a39e76ad4d203644d5 is 50, key is test_row_0/B:col10/1732742187113/Put/seqid=0 2024-11-27T21:16:28,392 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127178a65b2c5654be2a929cac6c8f46894_b6db78cfa2317a7fa812b5763434be10, store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:28,392 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127178a65b2c5654be2a929cac6c8f46894_b6db78cfa2317a7fa812b5763434be10 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:28,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742397_1573 (size=4469) 2024-11-27T21:16:28,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742396_1572 (size=12139) 2024-11-27T21:16:28,397 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#A#compaction#487 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:28,397 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/4f853468abf3420887ea9041f8e2c78e is 175, key is test_row_0/A:col10/1732742187113/Put/seqid=0 2024-11-27T21:16:28,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742398_1574 (size=31093) 2024-11-27T21:16:28,405 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/4f853468abf3420887ea9041f8e2c78e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/4f853468abf3420887ea9041f8e2c78e 2024-11-27T21:16:28,408 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/A of b6db78cfa2317a7fa812b5763434be10 into 4f853468abf3420887ea9041f8e2c78e(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:28,408 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:28,408 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/A, priority=12, startTime=1732742188378; duration=0sec 2024-11-27T21:16:28,408 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:28,408 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:A 2024-11-27T21:16:28,409 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:28,410 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:28,410 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/C is initiating minor compaction (all files) 2024-11-27T21:16:28,410 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/C in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:28,410 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/f031a49e63ee447c8b7b09fc763d949a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/617fe78eb1dc4c17b1afeef9aa90ad9b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/2c8fe9aaffb04b0a84b13d0c3259ba8a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/4c37337cfb8d406c9d18632588227697] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=46.9 K 2024-11-27T21:16:28,410 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting f031a49e63ee447c8b7b09fc763d949a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732742184330 2024-11-27T21:16:28,410 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 617fe78eb1dc4c17b1afeef9aa90ad9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732742184438 2024-11-27T21:16:28,411 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c8fe9aaffb04b0a84b13d0c3259ba8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732742185594 2024-11-27T21:16:28,411 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c37337cfb8d406c9d18632588227697, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732742186781 2024-11-27T21:16:28,417 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#C#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:28,417 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/8afb87bbc72d46338fb25a5c42622fcf is 50, key is test_row_0/C:col10/1732742187113/Put/seqid=0 2024-11-27T21:16:28,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742399_1575 (size=12139) 2024-11-27T21:16:28,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T21:16:28,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:28,528 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-27T21:16:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:28,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112707f4ca24e60549a69b42c5992dbb971b_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742187122/Put/seqid=0 2024-11-27T21:16:28,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742400_1576 (size=12154) 2024-11-27T21:16:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T21:16:28,801 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/a88d7bb8a60149a39e76ad4d203644d5 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/a88d7bb8a60149a39e76ad4d203644d5 2024-11-27T21:16:28,805 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/B of b6db78cfa2317a7fa812b5763434be10 into a88d7bb8a60149a39e76ad4d203644d5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:28,805 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:28,805 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/B, priority=12, startTime=1732742188378; duration=0sec 2024-11-27T21:16:28,805 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:28,805 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:B 2024-11-27T21:16:28,825 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/8afb87bbc72d46338fb25a5c42622fcf as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8afb87bbc72d46338fb25a5c42622fcf 2024-11-27T21:16:28,829 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/C of b6db78cfa2317a7fa812b5763434be10 into 8afb87bbc72d46338fb25a5c42622fcf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:28,829 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:28,829 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/C, priority=12, startTime=1732742188379; duration=0sec 2024-11-27T21:16:28,829 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:28,829 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:C 2024-11-27T21:16:28,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:28,943 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112707f4ca24e60549a69b42c5992dbb971b_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112707f4ca24e60549a69b42c5992dbb971b_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:28,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/f8882c37abf349cc900c3597e323f504, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:28,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/f8882c37abf349cc900c3597e323f504 is 175, key is test_row_0/A:col10/1732742187122/Put/seqid=0 2024-11-27T21:16:28,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742401_1577 (size=30955) 2024-11-27T21:16:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:29,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:29,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742189286, max timestamp=1732742189298Num uniques 859; 2024-11-27T21:16:29,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742189286, max timestamp=1732742189298Num uniques 859; 2024-11-27T21:16:29,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:16:29,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:16:29,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742189286, max timestamp=1732742189298Num uniques 859; 2024-11-27T21:16:29,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:16:29,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732742189307, max timestamp=1732742189325Num uniques 1068; 2024-11-27T21:16:29,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-27T21:16:29,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732742189307, max timestamp=1732742189325Num uniques 1068; 2024-11-27T21:16:29,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-27T21:16:29,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732742189307, max timestamp=1732742189325Num uniques 1068; 2024-11-27T21:16:29,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-27T21:16:29,350 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/f8882c37abf349cc900c3597e323f504 2024-11-27T21:16:29,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742249348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:29,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:29,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742249351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:29,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/1614ad86f1e04d509f32e9842fb29914 is 50, key is test_row_0/B:col10/1732742187122/Put/seqid=0 2024-11-27T21:16:29,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:29,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742402_1578 (size=12001) 2024-11-27T21:16:29,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742249353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742249355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:29,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742249356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:29,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:29,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:29,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742249459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:29,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742249459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:29,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:29,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742249469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742249469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:29,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:29,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742249469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:29,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:29,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:29,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:29,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742249669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:29,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742249669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:29,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742249679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:29,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742249679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:29,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742249679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T21:16:29,760 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/1614ad86f1e04d509f32e9842fb29914 2024-11-27T21:16:29,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/cfb340ba66a647c6b5a880ffc0dd06d3 is 50, key is test_row_0/C:col10/1732742187122/Put/seqid=0 2024-11-27T21:16:29,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742403_1579 (size=12001) 2024-11-27T21:16:29,775 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/cfb340ba66a647c6b5a880ffc0dd06d3 2024-11-27T21:16:29,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/f8882c37abf349cc900c3597e323f504 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/f8882c37abf349cc900c3597e323f504 2024-11-27T21:16:29,781 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/f8882c37abf349cc900c3597e323f504, entries=150, sequenceid=105, filesize=30.2 K 2024-11-27T21:16:29,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/1614ad86f1e04d509f32e9842fb29914 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/1614ad86f1e04d509f32e9842fb29914 2024-11-27T21:16:29,785 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/1614ad86f1e04d509f32e9842fb29914, entries=150, sequenceid=105, filesize=11.7 K 2024-11-27T21:16:29,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/cfb340ba66a647c6b5a880ffc0dd06d3 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/cfb340ba66a647c6b5a880ffc0dd06d3 2024-11-27T21:16:29,788 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/cfb340ba66a647c6b5a880ffc0dd06d3, entries=150, sequenceid=105, filesize=11.7 K 2024-11-27T21:16:29,789 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for b6db78cfa2317a7fa812b5763434be10 in 1261ms, sequenceid=105, compaction requested=false 2024-11-27T21:16:29,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:29,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:29,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-27T21:16:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-27T21:16:29,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-27T21:16:29,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1780 sec 2024-11-27T21:16:29,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.1820 sec 2024-11-27T21:16:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:30,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=161.02 KB heapSize=455.16 KB 2024-11-27T21:16:30,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:30,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:16:30,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:30,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:16:30,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:30,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:16:30,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742250001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742250001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742250002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742250002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742250005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fc476de89499427aacdd358217406eee_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742190000/Put/seqid=0 2024-11-27T21:16:30,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742404_1580 (size=14794) 2024-11-27T21:16:30,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742250118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742250119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742250119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742250119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742250119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742250329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742250329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742250329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742250329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742250329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,427 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,430 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fc476de89499427aacdd358217406eee_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fc476de89499427aacdd358217406eee_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:30,431 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/fbf328acbc6e44bbacd2719896b999c9, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:30,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/fbf328acbc6e44bbacd2719896b999c9 is 175, key is test_row_0/A:col10/1732742190000/Put/seqid=0 2024-11-27T21:16:30,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742405_1581 (size=39749) 2024-11-27T21:16:30,435 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/fbf328acbc6e44bbacd2719896b999c9 2024-11-27T21:16:30,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/c35074bb554a4e76bf3e2e454f0270a7 is 50, key is test_row_0/B:col10/1732742190000/Put/seqid=0 2024-11-27T21:16:30,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742406_1582 (size=12151) 2024-11-27T21:16:30,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742250639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742250639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742250639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742250640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742250640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:30,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/c35074bb554a4e76bf3e2e454f0270a7 2024-11-27T21:16:30,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/1e3cebab6499407ca3877b2af6f91ed7 is 50, key is test_row_0/C:col10/1732742190000/Put/seqid=0 2024-11-27T21:16:30,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742407_1583 (size=12151) 2024-11-27T21:16:30,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/1e3cebab6499407ca3877b2af6f91ed7 2024-11-27T21:16:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/fbf328acbc6e44bbacd2719896b999c9 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/fbf328acbc6e44bbacd2719896b999c9 2024-11-27T21:16:30,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/fbf328acbc6e44bbacd2719896b999c9, entries=200, sequenceid=165, filesize=38.8 K 2024-11-27T21:16:30,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/c35074bb554a4e76bf3e2e454f0270a7 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/c35074bb554a4e76bf3e2e454f0270a7 2024-11-27T21:16:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/c35074bb554a4e76bf3e2e454f0270a7, entries=150, sequenceid=165, filesize=11.9 K 2024-11-27T21:16:30,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/1e3cebab6499407ca3877b2af6f91ed7 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/1e3cebab6499407ca3877b2af6f91ed7 2024-11-27T21:16:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/1e3cebab6499407ca3877b2af6f91ed7, entries=150, sequenceid=165, filesize=11.9 K 2024-11-27T21:16:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~494.95 KB/506832, currentSize=6.71 KB/6870 for b6db78cfa2317a7fa812b5763434be10 in 882ms, sequenceid=165, compaction requested=true 2024-11-27T21:16:30,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:30,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:16:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:30,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:16:30,883 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:30,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:30,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:16:30,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,883 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,884 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:30,884 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,884 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/A is initiating minor compaction (all files) 2024-11-27T21:16:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,884 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/A in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:30,884 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/4f853468abf3420887ea9041f8e2c78e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/f8882c37abf349cc900c3597e323f504, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/fbf328acbc6e44bbacd2719896b999c9] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=99.4 K 2024-11-27T21:16:30,884 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:30,884 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/B is initiating minor compaction (all files) 2024-11-27T21:16:30,884 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/4f853468abf3420887ea9041f8e2c78e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/f8882c37abf349cc900c3597e323f504, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/fbf328acbc6e44bbacd2719896b999c9] 2024-11-27T21:16:30,884 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/B in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,884 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/a88d7bb8a60149a39e76ad4d203644d5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/1614ad86f1e04d509f32e9842fb29914, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/c35074bb554a4e76bf3e2e454f0270a7] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=35.4 K 2024-11-27T21:16:30,885 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f853468abf3420887ea9041f8e2c78e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732742186781 2024-11-27T21:16:30,885 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting a88d7bb8a60149a39e76ad4d203644d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732742186781 2024-11-27T21:16:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,885 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8882c37abf349cc900c3597e323f504, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732742187115 2024-11-27T21:16:30,885 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 1614ad86f1e04d509f32e9842fb29914, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732742187115 2024-11-27T21:16:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,885 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbf328acbc6e44bbacd2719896b999c9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732742189354 2024-11-27T21:16:30,885 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting c35074bb554a4e76bf3e2e454f0270a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732742189354 2024-11-27T21:16:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,890 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,892 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127cbfd08e8110a4f25ba88c4f900cf249e_b6db78cfa2317a7fa812b5763434be10 store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:30,892 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#B#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,892 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/d1368ac5933148bba9ce015918e9810e is 50, key is test_row_0/B:col10/1732742190000/Put/seqid=0 2024-11-27T21:16:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,893 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127cbfd08e8110a4f25ba88c4f900cf249e_b6db78cfa2317a7fa812b5763434be10, store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,894 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127cbfd08e8110a4f25ba88c4f900cf249e_b6db78cfa2317a7fa812b5763434be10 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742408_1584 (size=12391) 2024-11-27T21:16:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742409_1585 (size=4469) 2024-11-27T21:16:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,910 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#A#compaction#496 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,911 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/3690bd1f91eb4a39b5f9f01ef9db957c is 175, key is test_row_0/A:col10/1732742190000/Put/seqid=0 2024-11-27T21:16:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,913 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/d1368ac5933148bba9ce015918e9810e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d1368ac5933148bba9ce015918e9810e 2024-11-27T21:16:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742410_1586 (size=31345) 2024-11-27T21:16:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,918 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/B of b6db78cfa2317a7fa812b5763434be10 into d1368ac5933148bba9ce015918e9810e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,918 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:30,918 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/3690bd1f91eb4a39b5f9f01ef9db957c as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/3690bd1f91eb4a39b5f9f01ef9db957c 2024-11-27T21:16:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,918 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/B, priority=13, startTime=1732742190883; duration=0sec 2024-11-27T21:16:30,918 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:30,918 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:B 2024-11-27T21:16:30,918 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,919 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,919 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/C is initiating minor compaction (all files) 2024-11-27T21:16:30,920 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/C in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:30,920 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8afb87bbc72d46338fb25a5c42622fcf, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/cfb340ba66a647c6b5a880ffc0dd06d3, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/1e3cebab6499407ca3877b2af6f91ed7] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=35.4 K 2024-11-27T21:16:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,920 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 8afb87bbc72d46338fb25a5c42622fcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732742186781 2024-11-27T21:16:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,920 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting cfb340ba66a647c6b5a880ffc0dd06d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732742187115 2024-11-27T21:16:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,921 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e3cebab6499407ca3877b2af6f91ed7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732742189354 2024-11-27T21:16:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,922 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/A of b6db78cfa2317a7fa812b5763434be10 into 3690bd1f91eb4a39b5f9f01ef9db957c(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:30,922 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:30,922 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/A, priority=13, startTime=1732742190883; duration=0sec 2024-11-27T21:16:30,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,923 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:30,923 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:A 2024-11-27T21:16:30,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,929 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#C#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,929 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/6eee38d0ab664f1bbeda108fd9da9a5d is 50, key is test_row_0/C:col10/1732742190000/Put/seqid=0 2024-11-27T21:16:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742411_1587 (size=12391) 2024-11-27T21:16:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,938 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/6eee38d0ab664f1bbeda108fd9da9a5d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/6eee38d0ab664f1bbeda108fd9da9a5d 2024-11-27T21:16:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,943 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/C of b6db78cfa2317a7fa812b5763434be10 into 6eee38d0ab664f1bbeda108fd9da9a5d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,943 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:30,943 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/C, priority=13, startTime=1732742190883; duration=0sec 2024-11-27T21:16:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,943 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:30,943 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:C 2024-11-27T21:16:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:31,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:16:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:31,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:31,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:31,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:31,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:31,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127abb9e1d15cb442ea8a80eebba8fec747_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742191159/Put/seqid=0 2024-11-27T21:16:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742412_1588 (size=22268) 2024-11-27T21:16:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,175 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:31,179 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127abb9e1d15cb442ea8a80eebba8fec747_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127abb9e1d15cb442ea8a80eebba8fec747_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:31,180 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/307d9e124a5c4d3d8af89827607aeec0, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:31,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/307d9e124a5c4d3d8af89827607aeec0 is 175, key is test_row_0/A:col10/1732742191159/Put/seqid=0 2024-11-27T21:16:31,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742413_1589 (size=65673) 2024-11-27T21:16:31,184 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/307d9e124a5c4d3d8af89827607aeec0 2024-11-27T21:16:31,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/07cf68e3551442fda3cfae5f732de2c4 is 50, key is test_row_0/B:col10/1732742191159/Put/seqid=0 2024-11-27T21:16:31,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742191182, max timestamp=1732742191191Num uniques 859; 2024-11-27T21:16:31,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:16:31,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742191182, max timestamp=1732742191191Num uniques 859; 2024-11-27T21:16:31,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742191182, max timestamp=1732742191191Num uniques 859; 2024-11-27T21:16:31,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:16:31,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:16:31,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742414_1590 (size=12151) 2024-11-27T21:16:31,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742251212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742251216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742251218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742251219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742251219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742251329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742251329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742251329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742251329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742251329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742251539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742251539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742251539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742251539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742251539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/07cf68e3551442fda3cfae5f732de2c4 2024-11-27T21:16:31,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/fbc3c9bb47d34ad98f7289a5cdc77958 is 50, key is test_row_0/C:col10/1732742191159/Put/seqid=0 2024-11-27T21:16:31,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742415_1591 (size=12151) 2024-11-27T21:16:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T21:16:31,779 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:31,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:16:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-27T21:16:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T21:16:31,781 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:16:31,782 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:16:31,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:16:31,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742251849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742251849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742251849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742251849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742251849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:31,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:31,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:31,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T21:16:31,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-27T21:16:31,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:31,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:31,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:31,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:31,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:31,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:31,940 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T21:16:31,940 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-27T21:16:32,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/fbc3c9bb47d34ad98f7289a5cdc77958 2024-11-27T21:16:32,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/307d9e124a5c4d3d8af89827607aeec0 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/307d9e124a5c4d3d8af89827607aeec0 2024-11-27T21:16:32,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/307d9e124a5c4d3d8af89827607aeec0, entries=350, sequenceid=178, filesize=64.1 K 2024-11-27T21:16:32,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/07cf68e3551442fda3cfae5f732de2c4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/07cf68e3551442fda3cfae5f732de2c4 2024-11-27T21:16:32,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/07cf68e3551442fda3cfae5f732de2c4, entries=150, sequenceid=178, filesize=11.9 K 2024-11-27T21:16:32,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/fbc3c9bb47d34ad98f7289a5cdc77958 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/fbc3c9bb47d34ad98f7289a5cdc77958 2024-11-27T21:16:32,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/fbc3c9bb47d34ad98f7289a5cdc77958, entries=150, sequenceid=178, filesize=11.9 K 2024-11-27T21:16:32,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for b6db78cfa2317a7fa812b5763434be10 in 863ms, sequenceid=178, compaction requested=false 2024-11-27T21:16:32,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:32,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-27T21:16:32,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:32,087 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-27T21:16:32,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:32,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:32,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:32,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:32,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:32,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:32,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127cd3e9786bdf54bd2b478dfb1c6b5db84_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742191217/Put/seqid=0 2024-11-27T21:16:32,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T21:16:32,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742416_1592 (size=12304) 2024-11-27T21:16:32,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,109 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127cd3e9786bdf54bd2b478dfb1c6b5db84_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cd3e9786bdf54bd2b478dfb1c6b5db84_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:32,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/358de47f83674e97919957c278508ceb, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:32,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/358de47f83674e97919957c278508ceb is 175, key is test_row_0/A:col10/1732742191217/Put/seqid=0 2024-11-27T21:16:32,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742417_1593 (size=31105) 2024-11-27T21:16:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:32,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:32,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:32,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742252365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:32,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742252366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:32,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:32,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742252366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:32,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:32,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742252366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:32,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:32,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742252372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:32,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T21:16:32,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742252479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:32,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:32,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742252479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:32,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:32,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742252479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:32,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:32,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742252479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:32,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742252489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:32,515 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/358de47f83674e97919957c278508ceb 2024-11-27T21:16:32,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/d48a73c1134e4be381e95f47ee98c6b8 is 50, key is test_row_0/B:col10/1732742191217/Put/seqid=0 2024-11-27T21:16:32,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742418_1594 (size=12151) 2024-11-27T21:16:32,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742252689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:32,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742252689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:32,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742252689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:32,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742252689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:32,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742252699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:32,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:32,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T21:16:32,933 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/d48a73c1134e4be381e95f47ee98c6b8 2024-11-27T21:16:32,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/da4b6f1e7be4425797634a188e2addc6 is 50, key is test_row_0/C:col10/1732742191217/Put/seqid=0 2024-11-27T21:16:32,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742419_1595 (size=12151) 2024-11-27T21:16:32,957 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/da4b6f1e7be4425797634a188e2addc6 2024-11-27T21:16:32,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/358de47f83674e97919957c278508ceb as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/358de47f83674e97919957c278508ceb 2024-11-27T21:16:32,967 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/358de47f83674e97919957c278508ceb, entries=150, sequenceid=215, filesize=30.4 K 2024-11-27T21:16:32,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/d48a73c1134e4be381e95f47ee98c6b8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d48a73c1134e4be381e95f47ee98c6b8 2024-11-27T21:16:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,971 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d48a73c1134e4be381e95f47ee98c6b8, entries=150, sequenceid=215, filesize=11.9 K 2024-11-27T21:16:32,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/da4b6f1e7be4425797634a188e2addc6 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/da4b6f1e7be4425797634a188e2addc6 2024-11-27T21:16:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,976 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/da4b6f1e7be4425797634a188e2addc6, entries=150, sequenceid=215, filesize=11.9 K 2024-11-27T21:16:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,977 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for b6db78cfa2317a7fa812b5763434be10 in 890ms, sequenceid=215, compaction requested=true 2024-11-27T21:16:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-27T21:16:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-27T21:16:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-27T21:16:32,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1960 sec 2024-11-27T21:16:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.2010 sec 2024-11-27T21:16:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:33,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:16:33,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:33,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:33,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:33,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:33,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:33,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:33,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112772929aca399c496c8e905a9e2149ce53_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742193001/Put/seqid=0 2024-11-27T21:16:33,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742420_1596 (size=9814) 2024-11-27T21:16:33,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742193020, max timestamp=1732742193027Num uniques 859; 2024-11-27T21:16:33,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:33,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742193020, max timestamp=1732742193027Num uniques 859; 2024-11-27T21:16:33,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:33,031 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:33,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732742193020, max timestamp=1732742193027Num uniques 859; 2024-11-27T21:16:33,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T21:16:33,034 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112772929aca399c496c8e905a9e2149ce53_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112772929aca399c496c8e905a9e2149ce53_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:33,035 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/fbdb360f1a224ce29b2456e8562fdd53, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:33,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/fbdb360f1a224ce29b2456e8562fdd53 is 175, key is test_row_0/A:col10/1732742193001/Put/seqid=0 2024-11-27T21:16:33,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742421_1597 (size=22461) 2024-11-27T21:16:33,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742253051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742253051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742253057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742253058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742253059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742253169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742253169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742253169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742253169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742253170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742253379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742253379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742253379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742253380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742253380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,441 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/fbdb360f1a224ce29b2456e8562fdd53 2024-11-27T21:16:33,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/bb5c7f269a23417aa5d188fa9dfa53fc is 50, key is test_row_0/B:col10/1732742193001/Put/seqid=0 2024-11-27T21:16:33,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742422_1598 (size=9757) 2024-11-27T21:16:33,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/bb5c7f269a23417aa5d188fa9dfa53fc 2024-11-27T21:16:33,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/e3779042807c4249b7b2ff33c341b149 is 50, key is test_row_0/C:col10/1732742193001/Put/seqid=0 2024-11-27T21:16:33,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742423_1599 (size=9757) 2024-11-27T21:16:33,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/e3779042807c4249b7b2ff33c341b149 2024-11-27T21:16:33,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/fbdb360f1a224ce29b2456e8562fdd53 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/fbdb360f1a224ce29b2456e8562fdd53 2024-11-27T21:16:33,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/fbdb360f1a224ce29b2456e8562fdd53, entries=100, sequenceid=225, filesize=21.9 K 2024-11-27T21:16:33,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/bb5c7f269a23417aa5d188fa9dfa53fc as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/bb5c7f269a23417aa5d188fa9dfa53fc 2024-11-27T21:16:33,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/bb5c7f269a23417aa5d188fa9dfa53fc, entries=100, sequenceid=225, filesize=9.5 K 2024-11-27T21:16:33,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/e3779042807c4249b7b2ff33c341b149 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/e3779042807c4249b7b2ff33c341b149 2024-11-27T21:16:33,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/e3779042807c4249b7b2ff33c341b149, entries=100, sequenceid=225, filesize=9.5 K 2024-11-27T21:16:33,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b6db78cfa2317a7fa812b5763434be10 in 546ms, sequenceid=225, compaction requested=true 2024-11-27T21:16:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:33,549 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:16:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:16:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:16:33,549 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:33,551 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150584 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:33,551 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/A is initiating minor compaction (all files) 2024-11-27T21:16:33,551 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/A in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:33,551 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/3690bd1f91eb4a39b5f9f01ef9db957c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/307d9e124a5c4d3d8af89827607aeec0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/358de47f83674e97919957c278508ceb, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/fbdb360f1a224ce29b2456e8562fdd53] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=147.1 K 2024-11-27T21:16:33,551 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:33,551 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/3690bd1f91eb4a39b5f9f01ef9db957c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/307d9e124a5c4d3d8af89827607aeec0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/358de47f83674e97919957c278508ceb, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/fbdb360f1a224ce29b2456e8562fdd53] 2024-11-27T21:16:33,552 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:33,552 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/B is initiating minor compaction (all files) 2024-11-27T21:16:33,552 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/B in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:33,552 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d1368ac5933148bba9ce015918e9810e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/07cf68e3551442fda3cfae5f732de2c4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d48a73c1134e4be381e95f47ee98c6b8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/bb5c7f269a23417aa5d188fa9dfa53fc] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=45.4 K 2024-11-27T21:16:33,553 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3690bd1f91eb4a39b5f9f01ef9db957c, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732742189354 2024-11-27T21:16:33,553 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting d1368ac5933148bba9ce015918e9810e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732742189354 2024-11-27T21:16:33,554 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 307d9e124a5c4d3d8af89827607aeec0, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732742190002 2024-11-27T21:16:33,554 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 07cf68e3551442fda3cfae5f732de2c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732742191156 2024-11-27T21:16:33,554 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 358de47f83674e97919957c278508ceb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732742191215 2024-11-27T21:16:33,554 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting d48a73c1134e4be381e95f47ee98c6b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732742191215 2024-11-27T21:16:33,554 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbdb360f1a224ce29b2456e8562fdd53, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732742192365 2024-11-27T21:16:33,555 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting bb5c7f269a23417aa5d188fa9dfa53fc, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732742192365 2024-11-27T21:16:33,569 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:33,571 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#B#compaction#509 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:33,572 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/6c4cd7e7ab444bc883cf1aa7fae59b71 is 50, key is test_row_0/B:col10/1732742193001/Put/seqid=0 2024-11-27T21:16:33,577 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127243bd6bdf79f4c79b048a31d19dc9f65_b6db78cfa2317a7fa812b5763434be10 store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:33,579 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127243bd6bdf79f4c79b048a31d19dc9f65_b6db78cfa2317a7fa812b5763434be10, store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:33,580 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127243bd6bdf79f4c79b048a31d19dc9f65_b6db78cfa2317a7fa812b5763434be10 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:33,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742424_1600 (size=12527) 2024-11-27T21:16:33,595 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/6c4cd7e7ab444bc883cf1aa7fae59b71 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/6c4cd7e7ab444bc883cf1aa7fae59b71 2024-11-27T21:16:33,601 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/B of b6db78cfa2317a7fa812b5763434be10 into 6c4cd7e7ab444bc883cf1aa7fae59b71(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:33,601 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:33,602 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/B, priority=12, startTime=1732742193549; duration=0sec 2024-11-27T21:16:33,602 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:33,602 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:B 2024-11-27T21:16:33,602 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:33,604 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:33,604 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/C is initiating minor compaction (all files) 2024-11-27T21:16:33,604 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/C in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:33,604 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/6eee38d0ab664f1bbeda108fd9da9a5d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/fbc3c9bb47d34ad98f7289a5cdc77958, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/da4b6f1e7be4425797634a188e2addc6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/e3779042807c4249b7b2ff33c341b149] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=45.4 K 2024-11-27T21:16:33,605 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eee38d0ab664f1bbeda108fd9da9a5d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732742189354 2024-11-27T21:16:33,605 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting fbc3c9bb47d34ad98f7289a5cdc77958, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732742191156 2024-11-27T21:16:33,605 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting da4b6f1e7be4425797634a188e2addc6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732742191215 2024-11-27T21:16:33,613 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting e3779042807c4249b7b2ff33c341b149, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732742192365 2024-11-27T21:16:33,631 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#C#compaction#510 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:33,632 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/cbb9a5e1d88a45759f2bc9c88ab34059 is 50, key is test_row_0/C:col10/1732742193001/Put/seqid=0 2024-11-27T21:16:33,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742425_1601 (size=4469) 2024-11-27T21:16:33,640 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#A#compaction#508 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:33,641 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/9fe5ff8755cb47d3a19a583f78b66b1d is 175, key is test_row_0/A:col10/1732742193001/Put/seqid=0 2024-11-27T21:16:33,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742426_1602 (size=12527) 2024-11-27T21:16:33,652 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/cbb9a5e1d88a45759f2bc9c88ab34059 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/cbb9a5e1d88a45759f2bc9c88ab34059 2024-11-27T21:16:33,658 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/C of b6db78cfa2317a7fa812b5763434be10 into cbb9a5e1d88a45759f2bc9c88ab34059(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:33,658 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:33,658 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/C, priority=12, startTime=1732742193549; duration=0sec 2024-11-27T21:16:33,658 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:33,658 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:C 2024-11-27T21:16:33,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742427_1603 (size=31588) 2024-11-27T21:16:33,686 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/9fe5ff8755cb47d3a19a583f78b66b1d as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/9fe5ff8755cb47d3a19a583f78b66b1d 2024-11-27T21:16:33,691 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/A of b6db78cfa2317a7fa812b5763434be10 into 9fe5ff8755cb47d3a19a583f78b66b1d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:33,691 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:33,691 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/A, priority=12, startTime=1732742193549; duration=0sec 2024-11-27T21:16:33,692 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:33,692 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:A 2024-11-27T21:16:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:33,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-27T21:16:33,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:33,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:33,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:33,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:33,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:33,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:33,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742253701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742253706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273de076dc2bc84b9fbcd22117e645464d_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742193695/Put/seqid=0 2024-11-27T21:16:33,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742253708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742253710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742253712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742428_1604 (size=14894) 2024-11-27T21:16:33,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742253819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742253819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742253819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742253829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742253829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T21:16:33,930 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:33,931 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:16:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-27T21:16:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T21:16:33,937 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:16:33,939 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:16:33,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:16:34,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742254029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742254029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742254029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T21:16:34,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742254039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742254039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T21:16:34,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:34,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:34,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:34,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:34,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:34,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:34,129 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:34,132 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273de076dc2bc84b9fbcd22117e645464d_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273de076dc2bc84b9fbcd22117e645464d_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:34,133 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/d27a2d4db52b45c49c4319152246780b, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:34,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/d27a2d4db52b45c49c4319152246780b is 175, key is test_row_0/A:col10/1732742193695/Put/seqid=0 2024-11-27T21:16:34,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742429_1605 (size=39849) 2024-11-27T21:16:34,144 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/d27a2d4db52b45c49c4319152246780b 2024-11-27T21:16:34,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/d66ade8713f44b409136be42b3f70ef4 is 50, key is test_row_0/B:col10/1732742193695/Put/seqid=0 2024-11-27T21:16:34,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742430_1606 (size=12201) 2024-11-27T21:16:34,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/d66ade8713f44b409136be42b3f70ef4 2024-11-27T21:16:34,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/6662ff5c10d84e9db5ae78976c7d46cf is 50, key is test_row_0/C:col10/1732742193695/Put/seqid=0 2024-11-27T21:16:34,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742431_1607 (size=12201) 2024-11-27T21:16:34,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T21:16:34,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:34,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:34,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:34,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:34,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:34,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:34,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T21:16:34,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742254339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742254339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742254339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742254349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742254349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T21:16:34,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:34,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:34,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:34,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:34,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:34,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:34,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T21:16:34,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:34,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:34,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:34,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:34,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:34,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T21:16:34,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/6662ff5c10d84e9db5ae78976c7d46cf 2024-11-27T21:16:34,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/d27a2d4db52b45c49c4319152246780b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/d27a2d4db52b45c49c4319152246780b 2024-11-27T21:16:34,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/d27a2d4db52b45c49c4319152246780b, entries=200, sequenceid=264, filesize=38.9 K 2024-11-27T21:16:34,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/d66ade8713f44b409136be42b3f70ef4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d66ade8713f44b409136be42b3f70ef4 2024-11-27T21:16:34,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d66ade8713f44b409136be42b3f70ef4, entries=150, sequenceid=264, filesize=11.9 K 2024-11-27T21:16:34,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/6662ff5c10d84e9db5ae78976c7d46cf as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/6662ff5c10d84e9db5ae78976c7d46cf 2024-11-27T21:16:34,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/6662ff5c10d84e9db5ae78976c7d46cf, entries=150, sequenceid=264, filesize=11.9 K 2024-11-27T21:16:34,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for b6db78cfa2317a7fa812b5763434be10 in 923ms, sequenceid=264, compaction requested=false 2024-11-27T21:16:34,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:34,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T21:16:34,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:34,707 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:16:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:34,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411270072d20fbb7440fe99d7d495f8e3595c_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742193709/Put/seqid=0 2024-11-27T21:16:34,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742432_1608 (size=12454) 2024-11-27T21:16:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:34,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:34,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742254874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742254874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742254878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742194872, max timestamp=1732742194878Num uniques 859; 2024-11-27T21:16:34,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:16:34,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742194872, max timestamp=1732742194878Num uniques 859; 2024-11-27T21:16:34,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:16:34,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742194872, max timestamp=1732742194878Num uniques 859; 2024-11-27T21:16:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:16:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742254941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742254943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742254989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742254990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742254990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:35,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742255059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:35,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742255059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T21:16:35,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:35,131 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411270072d20fbb7440fe99d7d495f8e3595c_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270072d20fbb7440fe99d7d495f8e3595c_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:35,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/3b2fbfa0f2a54c43a68573e10ffa0781, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:35,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/3b2fbfa0f2a54c43a68573e10ffa0781 is 175, key is test_row_0/A:col10/1732742193709/Put/seqid=0 2024-11-27T21:16:35,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742433_1609 (size=31255) 2024-11-27T21:16:35,159 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/3b2fbfa0f2a54c43a68573e10ffa0781 2024-11-27T21:16:35,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/5bb452c397d742a18eec4e9966d2b2bf is 50, key is test_row_0/B:col10/1732742193709/Put/seqid=0 2024-11-27T21:16:35,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742434_1610 (size=12301) 2024-11-27T21:16:35,198 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/5bb452c397d742a18eec4e9966d2b2bf 2024-11-27T21:16:35,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742255199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:35,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:35,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742255199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742255200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/507e7652d85c4ef5a7be69fb039fbf7e is 50, key is test_row_0/C:col10/1732742193709/Put/seqid=0 2024-11-27T21:16:35,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742435_1611 (size=12301) 2024-11-27T21:16:35,213 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/507e7652d85c4ef5a7be69fb039fbf7e 2024-11-27T21:16:35,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/3b2fbfa0f2a54c43a68573e10ffa0781 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/3b2fbfa0f2a54c43a68573e10ffa0781 2024-11-27T21:16:35,225 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/3b2fbfa0f2a54c43a68573e10ffa0781, entries=150, sequenceid=274, filesize=30.5 K 2024-11-27T21:16:35,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/5bb452c397d742a18eec4e9966d2b2bf as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/5bb452c397d742a18eec4e9966d2b2bf 2024-11-27T21:16:35,231 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/5bb452c397d742a18eec4e9966d2b2bf, entries=150, sequenceid=274, filesize=12.0 K 2024-11-27T21:16:35,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/507e7652d85c4ef5a7be69fb039fbf7e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/507e7652d85c4ef5a7be69fb039fbf7e 2024-11-27T21:16:35,240 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/507e7652d85c4ef5a7be69fb039fbf7e, entries=150, sequenceid=274, filesize=12.0 K 2024-11-27T21:16:35,241 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b6db78cfa2317a7fa812b5763434be10 in 534ms, sequenceid=274, compaction requested=true 2024-11-27T21:16:35,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:35,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:35,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-27T21:16:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-27T21:16:35,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-27T21:16:35,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3040 sec 2024-11-27T21:16:35,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.3150 sec 2024-11-27T21:16:35,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=147.60 KB heapSize=425.53 KB 2024-11-27T21:16:35,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:35,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:16:35,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:35,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:16:35,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:35,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T21:16:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:35,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112739d53ac69a774bcc85dce2c7657b6548_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742194941/Put/seqid=0 2024-11-27T21:16:35,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742255297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:35,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:35,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742255305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:35,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:35,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742436_1612 (size=12454) 2024-11-27T21:16:35,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742255409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:35,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742255419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:35,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742255509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:35,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742255510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:35,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742255510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:35,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742255619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:35,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742255629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:35,722 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:35,725 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112739d53ac69a774bcc85dce2c7657b6548_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112739d53ac69a774bcc85dce2c7657b6548_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:35,726 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/47df32267bcb45ba9654b03d18ce57c1, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:35,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/47df32267bcb45ba9654b03d18ce57c1 is 175, key is test_row_0/A:col10/1732742194941/Put/seqid=0 2024-11-27T21:16:35,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742437_1613 (size=31255) 2024-11-27T21:16:35,755 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/47df32267bcb45ba9654b03d18ce57c1 2024-11-27T21:16:35,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/0721ba5b5c4f4c41ab0e8b431c4234f6 is 50, key is test_row_0/B:col10/1732742194941/Put/seqid=0 2024-11-27T21:16:35,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742438_1614 (size=12301) 2024-11-27T21:16:35,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/0721ba5b5c4f4c41ab0e8b431c4234f6 2024-11-27T21:16:35,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/c2e2f9375d84460f96eafd9bdfbd067e is 50, key is test_row_0/C:col10/1732742194941/Put/seqid=0 2024-11-27T21:16:35,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742439_1615 (size=12301) 2024-11-27T21:16:35,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/c2e2f9375d84460f96eafd9bdfbd067e 2024-11-27T21:16:35,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/47df32267bcb45ba9654b03d18ce57c1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/47df32267bcb45ba9654b03d18ce57c1 2024-11-27T21:16:35,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/47df32267bcb45ba9654b03d18ce57c1, entries=150, sequenceid=313, filesize=30.5 K 2024-11-27T21:16:35,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/0721ba5b5c4f4c41ab0e8b431c4234f6 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/0721ba5b5c4f4c41ab0e8b431c4234f6 2024-11-27T21:16:35,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/0721ba5b5c4f4c41ab0e8b431c4234f6, entries=150, sequenceid=313, filesize=12.0 K 2024-11-27T21:16:35,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/c2e2f9375d84460f96eafd9bdfbd067e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/c2e2f9375d84460f96eafd9bdfbd067e 2024-11-27T21:16:35,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/c2e2f9375d84460f96eafd9bdfbd067e, entries=150, sequenceid=313, filesize=12.0 K 2024-11-27T21:16:35,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=33.54 KB/34350 for b6db78cfa2317a7fa812b5763434be10 in 539ms, sequenceid=313, compaction requested=true 2024-11-27T21:16:35,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:35,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:16:35,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:35,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:16:35,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:35,814 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:35,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:16:35,814 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:35,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:35,815 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133947 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:35,815 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49330 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:35,815 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/A is initiating minor compaction (all files) 2024-11-27T21:16:35,815 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/B is initiating minor compaction (all files) 2024-11-27T21:16:35,815 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/A in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:35,816 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/B in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:35,816 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/9fe5ff8755cb47d3a19a583f78b66b1d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/d27a2d4db52b45c49c4319152246780b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/3b2fbfa0f2a54c43a68573e10ffa0781, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/47df32267bcb45ba9654b03d18ce57c1] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=130.8 K 2024-11-27T21:16:35,816 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:35,816 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/6c4cd7e7ab444bc883cf1aa7fae59b71, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d66ade8713f44b409136be42b3f70ef4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/5bb452c397d742a18eec4e9966d2b2bf, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/0721ba5b5c4f4c41ab0e8b431c4234f6] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=48.2 K 2024-11-27T21:16:35,816 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/9fe5ff8755cb47d3a19a583f78b66b1d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/d27a2d4db52b45c49c4319152246780b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/3b2fbfa0f2a54c43a68573e10ffa0781, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/47df32267bcb45ba9654b03d18ce57c1] 2024-11-27T21:16:35,817 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c4cd7e7ab444bc883cf1aa7fae59b71, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732742191216 2024-11-27T21:16:35,817 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fe5ff8755cb47d3a19a583f78b66b1d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732742191216 2024-11-27T21:16:35,818 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting d66ade8713f44b409136be42b3f70ef4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732742193038 2024-11-27T21:16:35,818 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting d27a2d4db52b45c49c4319152246780b, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732742193038 2024-11-27T21:16:35,818 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bb452c397d742a18eec4e9966d2b2bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732742193706 2024-11-27T21:16:35,818 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b2fbfa0f2a54c43a68573e10ffa0781, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732742193706 2024-11-27T21:16:35,819 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47df32267bcb45ba9654b03d18ce57c1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732742194941 2024-11-27T21:16:35,819 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 0721ba5b5c4f4c41ab0e8b431c4234f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732742194941 2024-11-27T21:16:35,837 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:35,848 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#B#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:35,848 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/0ed5cd95234749c59379a07d28a45ec4 is 50, key is test_row_0/B:col10/1732742194941/Put/seqid=0 2024-11-27T21:16:35,853 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411277886368b34964096805ab09cde1e31f7_b6db78cfa2317a7fa812b5763434be10 store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:35,855 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411277886368b34964096805ab09cde1e31f7_b6db78cfa2317a7fa812b5763434be10, store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:35,855 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277886368b34964096805ab09cde1e31f7_b6db78cfa2317a7fa812b5763434be10 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:35,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742440_1616 (size=12813) 2024-11-27T21:16:35,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742441_1617 (size=4469) 2024-11-27T21:16:35,930 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#A#compaction#520 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:35,931 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/e22fdf004a704ba8acb59a3320be618f is 175, key is test_row_0/A:col10/1732742194941/Put/seqid=0 2024-11-27T21:16:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:35,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T21:16:35,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:35,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:35,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:35,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:35,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:35,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:35,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742442_1618 (size=31767) 2024-11-27T21:16:35,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a6532e323ff14f37a4147b905f2517e9_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742195931/Put/seqid=0 2024-11-27T21:16:35,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742443_1619 (size=14994) 2024-11-27T21:16:36,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742195983, max timestamp=1732742195997Num uniques 859; 2024-11-27T21:16:36,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742195983, max timestamp=1732742195997Num uniques 859; 2024-11-27T21:16:36,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:16:36,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:16:36,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732742195983, max timestamp=1732742195997Num uniques 859; 2024-11-27T21:16:36,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T21:16:36,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742256039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742256040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742256042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742256045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742256047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T21:16:36,079 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:36,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:16:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-27T21:16:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T21:16:36,082 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:16:36,085 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:16:36,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:16:36,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742256160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742256160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742256160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742256160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742256160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T21:16:36,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T21:16:36,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:36,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:36,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:36,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:36,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:36,284 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/0ed5cd95234749c59379a07d28a45ec4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/0ed5cd95234749c59379a07d28a45ec4 2024-11-27T21:16:36,289 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/B of b6db78cfa2317a7fa812b5763434be10 into 0ed5cd95234749c59379a07d28a45ec4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:36,289 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:36,289 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/B, priority=12, startTime=1732742195814; duration=0sec 2024-11-27T21:16:36,289 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:36,289 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:B 2024-11-27T21:16:36,289 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:36,292 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49330 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:36,293 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/C is initiating minor compaction (all files) 2024-11-27T21:16:36,293 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/C in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:36,293 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/cbb9a5e1d88a45759f2bc9c88ab34059, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/6662ff5c10d84e9db5ae78976c7d46cf, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/507e7652d85c4ef5a7be69fb039fbf7e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/c2e2f9375d84460f96eafd9bdfbd067e] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=48.2 K 2024-11-27T21:16:36,294 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting cbb9a5e1d88a45759f2bc9c88ab34059, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732742191216 2024-11-27T21:16:36,294 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 6662ff5c10d84e9db5ae78976c7d46cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732742193038 2024-11-27T21:16:36,295 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 507e7652d85c4ef5a7be69fb039fbf7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732742193706 2024-11-27T21:16:36,295 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting c2e2f9375d84460f96eafd9bdfbd067e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732742194941 2024-11-27T21:16:36,307 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#C#compaction#523 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:36,308 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/8dc3e78316834ad2aab6f60c8593ebe3 is 50, key is test_row_0/C:col10/1732742194941/Put/seqid=0 2024-11-27T21:16:36,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742444_1620 (size=12813) 2024-11-27T21:16:36,336 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/8dc3e78316834ad2aab6f60c8593ebe3 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8dc3e78316834ad2aab6f60c8593ebe3 2024-11-27T21:16:36,341 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/C of b6db78cfa2317a7fa812b5763434be10 into 8dc3e78316834ad2aab6f60c8593ebe3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:36,341 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:36,341 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/C, priority=12, startTime=1732742195814; duration=0sec 2024-11-27T21:16:36,341 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:36,341 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:C 2024-11-27T21:16:36,356 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/e22fdf004a704ba8acb59a3320be618f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/e22fdf004a704ba8acb59a3320be618f 2024-11-27T21:16:36,358 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:36,362 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/A of b6db78cfa2317a7fa812b5763434be10 into e22fdf004a704ba8acb59a3320be618f(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:36,362 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:36,362 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/A, priority=12, startTime=1732742195814; duration=0sec 2024-11-27T21:16:36,362 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:36,362 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:A 2024-11-27T21:16:36,365 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a6532e323ff14f37a4147b905f2517e9_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a6532e323ff14f37a4147b905f2517e9_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:36,365 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/259b07cbf3774634acd0529944b5e7e9, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:36,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/259b07cbf3774634acd0529944b5e7e9 is 175, key is test_row_0/A:col10/1732742195931/Put/seqid=0 2024-11-27T21:16:36,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742445_1621 (size=39949) 2024-11-27T21:16:36,371 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/259b07cbf3774634acd0529944b5e7e9 2024-11-27T21:16:36,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742256369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742256369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742256369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742256369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742256370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/b4f333acf84244dd9084159d5eb17ffa is 50, key is test_row_0/B:col10/1732742195931/Put/seqid=0 2024-11-27T21:16:36,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T21:16:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:36,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T21:16:36,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742446_1622 (size=12301) 2024-11-27T21:16:36,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T21:16:36,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:36,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:36,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:36,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:36,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:36,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742256679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742256679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742256679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742256679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742256688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:36,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T21:16:36,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:36,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:36,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:36,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:36,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T21:16:36,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/b4f333acf84244dd9084159d5eb17ffa 2024-11-27T21:16:36,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/5f2fb194f15a445b8d6d7ec2b6a5cd17 is 50, key is test_row_0/C:col10/1732742195931/Put/seqid=0 2024-11-27T21:16:36,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742447_1623 (size=12301) 2024-11-27T21:16:36,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/5f2fb194f15a445b8d6d7ec2b6a5cd17 2024-11-27T21:16:36,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/259b07cbf3774634acd0529944b5e7e9 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/259b07cbf3774634acd0529944b5e7e9 2024-11-27T21:16:36,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/259b07cbf3774634acd0529944b5e7e9, entries=200, sequenceid=323, filesize=39.0 K 2024-11-27T21:16:36,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/b4f333acf84244dd9084159d5eb17ffa as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/b4f333acf84244dd9084159d5eb17ffa 2024-11-27T21:16:36,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/b4f333acf84244dd9084159d5eb17ffa, entries=150, sequenceid=323, filesize=12.0 K 2024-11-27T21:16:36,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/5f2fb194f15a445b8d6d7ec2b6a5cd17 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/5f2fb194f15a445b8d6d7ec2b6a5cd17 2024-11-27T21:16:36,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/5f2fb194f15a445b8d6d7ec2b6a5cd17, entries=150, sequenceid=323, filesize=12.0 K 2024-11-27T21:16:36,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b6db78cfa2317a7fa812b5763434be10 in 910ms, sequenceid=323, compaction requested=false 2024-11-27T21:16:36,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:36,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T21:16:36,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:36,853 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-27T21:16:36,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:36,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:36,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:36,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:36,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:36,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:36,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112799408590a432469ba439e6876e70bf78_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742196045/Put/seqid=0 2024-11-27T21:16:36,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742448_1624 (size=12454) 2024-11-27T21:16:36,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:36,868 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112799408590a432469ba439e6876e70bf78_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112799408590a432469ba439e6876e70bf78_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:36,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/5a9ff3f868ad408e857709d04b2def55, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:36,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/5a9ff3f868ad408e857709d04b2def55 is 175, key is test_row_0/A:col10/1732742196045/Put/seqid=0 2024-11-27T21:16:36,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742449_1625 (size=31255) 2024-11-27T21:16:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:37,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:37,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742257195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742257199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:37,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742257200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742257200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:37,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742257202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T21:16:37,275 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/5a9ff3f868ad408e857709d04b2def55 2024-11-27T21:16:37,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/4473c2748f874460a64808961f06b63f is 50, key is test_row_0/B:col10/1732742196045/Put/seqid=0 2024-11-27T21:16:37,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742450_1626 (size=12301) 2024-11-27T21:16:37,286 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/4473c2748f874460a64808961f06b63f 2024-11-27T21:16:37,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/d39d92f1d9474e188e1ddbcd623dbdcf is 50, key is test_row_0/C:col10/1732742196045/Put/seqid=0 2024-11-27T21:16:37,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742451_1627 (size=12301) 2024-11-27T21:16:37,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742257309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742257309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742257309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:37,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742257319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:37,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742257519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742257519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:37,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:37,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742257519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:37,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742257529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:37,698 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/d39d92f1d9474e188e1ddbcd623dbdcf 2024-11-27T21:16:37,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/5a9ff3f868ad408e857709d04b2def55 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5a9ff3f868ad408e857709d04b2def55 2024-11-27T21:16:37,705 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5a9ff3f868ad408e857709d04b2def55, entries=150, sequenceid=363, filesize=30.5 K 2024-11-27T21:16:37,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/4473c2748f874460a64808961f06b63f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/4473c2748f874460a64808961f06b63f 2024-11-27T21:16:37,708 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/4473c2748f874460a64808961f06b63f, entries=150, sequenceid=363, filesize=12.0 K 2024-11-27T21:16:37,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/d39d92f1d9474e188e1ddbcd623dbdcf as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/d39d92f1d9474e188e1ddbcd623dbdcf 2024-11-27T21:16:37,712 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/d39d92f1d9474e188e1ddbcd623dbdcf, entries=150, sequenceid=363, filesize=12.0 K 2024-11-27T21:16:37,713 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for b6db78cfa2317a7fa812b5763434be10 in 859ms, sequenceid=363, compaction requested=true 2024-11-27T21:16:37,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:37,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:37,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-27T21:16:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-27T21:16:37,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-27T21:16:37,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6290 sec 2024-11-27T21:16:37,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.6370 sec 2024-11-27T21:16:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:37,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-27T21:16:37,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:37,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:37,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:37,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:37,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:37,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:37,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112740e57c0364484bd1b01629f0f6e69517_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742197199/Put/seqid=0 2024-11-27T21:16:37,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742452_1628 (size=12454) 2024-11-27T21:16:37,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742197854, max timestamp=1732742197865Num uniques 859; 2024-11-27T21:16:37,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742197854, max timestamp=1732742197865Num uniques 859; 2024-11-27T21:16:37,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732742197854, max timestamp=1732742197865Num uniques 859; 2024-11-27T21:16:37,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:16:37,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:16:37,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T21:16:37,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742257893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742257893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742257894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:37,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742257899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:37,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:37,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:38,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742258009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:38,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742258009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:38,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742258009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:38,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742258010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:38,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:38,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742258219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:38,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742258219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:38,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742258219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,224 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_1' in b6db78cfa2317a7fa812b5763434be10 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:16:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742258220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:38,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:38,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742258220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T21:16:38,229 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:38,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:16:38,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-27T21:16:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T21:16:38,231 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:16:38,232 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:16:38,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:16:38,245 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:38,248 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112740e57c0364484bd1b01629f0f6e69517_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112740e57c0364484bd1b01629f0f6e69517_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:38,249 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/96777d3c2fc54e028c3071dd3c49514b, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:38,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/96777d3c2fc54e028c3071dd3c49514b is 175, key is test_row_0/A:col10/1732742197199/Put/seqid=0 2024-11-27T21:16:38,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742453_1629 (size=31255) 2024-11-27T21:16:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T21:16:38,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-27T21:16:38,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:38,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:38,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:38,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:38,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:38,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742258529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:38,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742258529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:38,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742258530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:38,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742258530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:38,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-27T21:16:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:38,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T21:16:38,653 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/96777d3c2fc54e028c3071dd3c49514b 2024-11-27T21:16:38,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/a617c6dfe81e4246b1b4cb4218c313d6 is 50, key is test_row_0/B:col10/1732742197199/Put/seqid=0 2024-11-27T21:16:38,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742454_1630 (size=12301) 2024-11-27T21:16:38,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-27T21:16:38,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:38,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:38,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:38,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:38,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:38,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-27T21:16:38,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:38,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:38,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:38,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:38,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T21:16:38,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-27T21:16:38,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:38,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:38,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:38,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:38,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:39,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:39,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742259039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:39,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:39,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:39,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:39,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742259049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:39,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742259049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:39,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742259049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:39,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/a617c6dfe81e4246b1b4cb4218c313d6 2024-11-27T21:16:39,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/ae3e7d8e309d402dbb47bd426fe01af4 is 50, key is test_row_0/C:col10/1732742197199/Put/seqid=0 2024-11-27T21:16:39,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742455_1631 (size=12301) 2024-11-27T21:16:39,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-27T21:16:39,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:39,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:39,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:39,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:39,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:39,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-27T21:16:39,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:39,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:39,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:39,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:39,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T21:16:39,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-27T21:16:39,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:39,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:39,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:39,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:39,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:39,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/ae3e7d8e309d402dbb47bd426fe01af4 2024-11-27T21:16:39,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/96777d3c2fc54e028c3071dd3c49514b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/96777d3c2fc54e028c3071dd3c49514b 2024-11-27T21:16:39,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/96777d3c2fc54e028c3071dd3c49514b, entries=150, sequenceid=374, filesize=30.5 K 2024-11-27T21:16:39,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/a617c6dfe81e4246b1b4cb4218c313d6 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/a617c6dfe81e4246b1b4cb4218c313d6 2024-11-27T21:16:39,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/a617c6dfe81e4246b1b4cb4218c313d6, entries=150, sequenceid=374, filesize=12.0 K 2024-11-27T21:16:39,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/ae3e7d8e309d402dbb47bd426fe01af4 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/ae3e7d8e309d402dbb47bd426fe01af4 2024-11-27T21:16:39,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/ae3e7d8e309d402dbb47bd426fe01af4, entries=150, sequenceid=374, filesize=12.0 K 2024-11-27T21:16:39,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b6db78cfa2317a7fa812b5763434be10 in 1656ms, sequenceid=374, compaction requested=true 2024-11-27T21:16:39,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:39,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:16:39,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:39,487 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:39,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:16:39,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:39,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:16:39,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:39,487 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:39,488 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:39,488 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:39,488 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/B is initiating minor compaction (all files) 2024-11-27T21:16:39,488 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/A is initiating minor compaction (all files) 2024-11-27T21:16:39,488 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/B in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:39,488 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/A in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:39,488 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/e22fdf004a704ba8acb59a3320be618f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/259b07cbf3774634acd0529944b5e7e9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5a9ff3f868ad408e857709d04b2def55, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/96777d3c2fc54e028c3071dd3c49514b] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=131.1 K 2024-11-27T21:16:39,488 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/0ed5cd95234749c59379a07d28a45ec4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/b4f333acf84244dd9084159d5eb17ffa, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/4473c2748f874460a64808961f06b63f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/a617c6dfe81e4246b1b4cb4218c313d6] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=48.6 K 2024-11-27T21:16:39,488 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:39,488 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/e22fdf004a704ba8acb59a3320be618f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/259b07cbf3774634acd0529944b5e7e9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5a9ff3f868ad408e857709d04b2def55, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/96777d3c2fc54e028c3071dd3c49514b] 2024-11-27T21:16:39,489 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting e22fdf004a704ba8acb59a3320be618f, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732742194941 2024-11-27T21:16:39,489 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ed5cd95234749c59379a07d28a45ec4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732742194941 2024-11-27T21:16:39,489 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 259b07cbf3774634acd0529944b5e7e9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732742195293 2024-11-27T21:16:39,489 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting b4f333acf84244dd9084159d5eb17ffa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732742195293 2024-11-27T21:16:39,490 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 4473c2748f874460a64808961f06b63f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732742196038 2024-11-27T21:16:39,490 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a9ff3f868ad408e857709d04b2def55, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732742196038 2024-11-27T21:16:39,490 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting a617c6dfe81e4246b1b4cb4218c313d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732742197199 2024-11-27T21:16:39,490 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96777d3c2fc54e028c3071dd3c49514b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732742197199 2024-11-27T21:16:39,503 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:39,504 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127effda5d7c80a44f1a1b213e5540ace45_b6db78cfa2317a7fa812b5763434be10 store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:39,505 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#B#compaction#533 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:39,505 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/8fa15c839b5049838fd76e86cf1cb90e is 50, key is test_row_0/B:col10/1732742197199/Put/seqid=0 2024-11-27T21:16:39,507 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127effda5d7c80a44f1a1b213e5540ace45_b6db78cfa2317a7fa812b5763434be10, store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:39,507 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127effda5d7c80a44f1a1b213e5540ace45_b6db78cfa2317a7fa812b5763434be10 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:39,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742456_1632 (size=12949) 2024-11-27T21:16:39,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742457_1633 (size=4469) 2024-11-27T21:16:39,523 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/8fa15c839b5049838fd76e86cf1cb90e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/8fa15c839b5049838fd76e86cf1cb90e 2024-11-27T21:16:39,523 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#A#compaction#532 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:39,524 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/214220df3e084630b4254a9ea9a18857 is 175, key is test_row_0/A:col10/1732742197199/Put/seqid=0 2024-11-27T21:16:39,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742458_1634 (size=31903) 2024-11-27T21:16:39,529 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/B of b6db78cfa2317a7fa812b5763434be10 into 8fa15c839b5049838fd76e86cf1cb90e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:39,529 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:39,529 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/B, priority=12, startTime=1732742199487; duration=0sec 2024-11-27T21:16:39,529 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:39,529 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:B 2024-11-27T21:16:39,529 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:39,531 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:39,531 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/C is initiating minor compaction (all files) 2024-11-27T21:16:39,531 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/C in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:39,531 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8dc3e78316834ad2aab6f60c8593ebe3, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/5f2fb194f15a445b8d6d7ec2b6a5cd17, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/d39d92f1d9474e188e1ddbcd623dbdcf, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/ae3e7d8e309d402dbb47bd426fe01af4] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=48.6 K 2024-11-27T21:16:39,532 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dc3e78316834ad2aab6f60c8593ebe3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732742194941 2024-11-27T21:16:39,533 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f2fb194f15a445b8d6d7ec2b6a5cd17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732742195293 2024-11-27T21:16:39,533 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/214220df3e084630b4254a9ea9a18857 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/214220df3e084630b4254a9ea9a18857 2024-11-27T21:16:39,533 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting d39d92f1d9474e188e1ddbcd623dbdcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732742196038 2024-11-27T21:16:39,533 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting ae3e7d8e309d402dbb47bd426fe01af4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732742197199 2024-11-27T21:16:39,538 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/A of b6db78cfa2317a7fa812b5763434be10 into 214220df3e084630b4254a9ea9a18857(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:39,538 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:39,538 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/A, priority=12, startTime=1732742199487; duration=0sec 2024-11-27T21:16:39,538 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:39,538 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:A 2024-11-27T21:16:39,541 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#C#compaction#534 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:39,541 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/7c34d29de0ce4a48b3bec22bbb7c0abd is 50, key is test_row_0/C:col10/1732742197199/Put/seqid=0 2024-11-27T21:16:39,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742459_1635 (size=12949) 2024-11-27T21:16:39,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-27T21:16:39,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:39,607 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-27T21:16:39,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:39,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:39,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:39,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:39,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:39,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:39,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fdc069a933c145b59c9316eca2ec98c4_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742197891/Put/seqid=0 2024-11-27T21:16:39,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742460_1636 (size=12454) 2024-11-27T21:16:39,949 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/7c34d29de0ce4a48b3bec22bbb7c0abd as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/7c34d29de0ce4a48b3bec22bbb7c0abd 2024-11-27T21:16:39,953 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/C of b6db78cfa2317a7fa812b5763434be10 into 7c34d29de0ce4a48b3bec22bbb7c0abd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:39,953 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:39,953 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/C, priority=12, startTime=1732742199487; duration=0sec 2024-11-27T21:16:39,953 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:39,953 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:C 2024-11-27T21:16:40,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:40,022 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fdc069a933c145b59c9316eca2ec98c4_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fdc069a933c145b59c9316eca2ec98c4_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:40,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/6b6c3dc5e7a24671a7f350f7e285fd23, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:40,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/6b6c3dc5e7a24671a7f350f7e285fd23 is 175, key is test_row_0/A:col10/1732742197891/Put/seqid=0 2024-11-27T21:16:40,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742461_1637 (size=31255) 2024-11-27T21:16:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:40,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:40,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742260075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742260075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742260076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742260078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742260189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742260189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742260189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742260189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742260239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,243 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_1' in b6db78cfa2317a7fa812b5763434be10 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:16:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T21:16:40,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742260399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742260399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742260400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742260400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,439 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=411, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/6b6c3dc5e7a24671a7f350f7e285fd23 2024-11-27T21:16:40,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/4930f3368f2f48d98a9ab60fd6f354a1 is 50, key is test_row_0/B:col10/1732742197891/Put/seqid=0 2024-11-27T21:16:40,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742462_1638 (size=12301) 2024-11-27T21:16:40,504 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T21:16:40,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742260709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742260709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742260709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742260709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:40,852 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/4930f3368f2f48d98a9ab60fd6f354a1 2024-11-27T21:16:40,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/508cca7576b041b19f32e9dee13f62d8 is 50, key is test_row_0/C:col10/1732742197891/Put/seqid=0 2024-11-27T21:16:40,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742463_1639 (size=12301) 2024-11-27T21:16:41,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742261218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:41,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742261219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:41,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742261219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:41,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:41,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742261229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:41,263 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/508cca7576b041b19f32e9dee13f62d8 2024-11-27T21:16:41,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/6b6c3dc5e7a24671a7f350f7e285fd23 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/6b6c3dc5e7a24671a7f350f7e285fd23 2024-11-27T21:16:41,269 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/6b6c3dc5e7a24671a7f350f7e285fd23, entries=150, sequenceid=411, filesize=30.5 K 2024-11-27T21:16:41,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/4930f3368f2f48d98a9ab60fd6f354a1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/4930f3368f2f48d98a9ab60fd6f354a1 2024-11-27T21:16:41,273 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/4930f3368f2f48d98a9ab60fd6f354a1, entries=150, sequenceid=411, filesize=12.0 K 2024-11-27T21:16:41,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/508cca7576b041b19f32e9dee13f62d8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/508cca7576b041b19f32e9dee13f62d8 2024-11-27T21:16:41,277 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/508cca7576b041b19f32e9dee13f62d8, entries=150, sequenceid=411, filesize=12.0 K 2024-11-27T21:16:41,278 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for b6db78cfa2317a7fa812b5763434be10 in 1672ms, sequenceid=411, compaction requested=false 2024-11-27T21:16:41,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:41,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:41,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-27T21:16:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-27T21:16:41,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-27T21:16:41,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0470 sec 2024-11-27T21:16:41,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 3.0510 sec 2024-11-27T21:16:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:42,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-27T21:16:42,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:42,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:42,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:42,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:42,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:42,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:42,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112770fba8acf9c14c958995b1d7e4340899_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742200075/Put/seqid=0 2024-11-27T21:16:42,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742464_1640 (size=14994) 2024-11-27T21:16:42,243 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:42,247 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112770fba8acf9c14c958995b1d7e4340899_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112770fba8acf9c14c958995b1d7e4340899_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:42,248 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/5fca7d1127b944e98788e7af9b24badb, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:42,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/5fca7d1127b944e98788e7af9b24badb is 175, key is test_row_0/A:col10/1732742200075/Put/seqid=0 2024-11-27T21:16:42,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742465_1641 (size=39949) 2024-11-27T21:16:42,252 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=424, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/5fca7d1127b944e98788e7af9b24badb 2024-11-27T21:16:42,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/db4703b6047443b995d6d99d3d0a038e is 50, key is test_row_0/B:col10/1732742200075/Put/seqid=0 2024-11-27T21:16:42,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742466_1642 (size=12301) 2024-11-27T21:16:42,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/db4703b6047443b995d6d99d3d0a038e 2024-11-27T21:16:42,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/69208914247e4906abd3f7f5d6e48fe8 is 50, key is test_row_0/C:col10/1732742200075/Put/seqid=0 2024-11-27T21:16:42,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742262265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:42,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742262265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742467_1643 (size=12301) 2024-11-27T21:16:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:42,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742262266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:42,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/69208914247e4906abd3f7f5d6e48fe8 2024-11-27T21:16:42,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/5fca7d1127b944e98788e7af9b24badb as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5fca7d1127b944e98788e7af9b24badb 2024-11-27T21:16:42,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:42,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742262270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:42,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5fca7d1127b944e98788e7af9b24badb, entries=200, sequenceid=424, filesize=39.0 K 2024-11-27T21:16:42,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/db4703b6047443b995d6d99d3d0a038e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/db4703b6047443b995d6d99d3d0a038e 2024-11-27T21:16:42,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/db4703b6047443b995d6d99d3d0a038e, entries=150, sequenceid=424, filesize=12.0 K 2024-11-27T21:16:42,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/69208914247e4906abd3f7f5d6e48fe8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/69208914247e4906abd3f7f5d6e48fe8 2024-11-27T21:16:42,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/69208914247e4906abd3f7f5d6e48fe8, entries=150, sequenceid=424, filesize=12.0 K 2024-11-27T21:16:42,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b6db78cfa2317a7fa812b5763434be10 in 53ms, sequenceid=424, compaction requested=true 2024-11-27T21:16:42,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:42,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:16:42,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:42,286 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:42,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:16:42,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:42,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:16:42,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:42,286 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:42,286 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:42,287 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:42,287 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/A is initiating minor compaction (all files) 2024-11-27T21:16:42,287 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/B is initiating minor compaction (all files) 2024-11-27T21:16:42,287 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/A in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:42,287 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/B in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:42,287 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/214220df3e084630b4254a9ea9a18857, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/6b6c3dc5e7a24671a7f350f7e285fd23, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5fca7d1127b944e98788e7af9b24badb] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=100.7 K 2024-11-27T21:16:42,287 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/8fa15c839b5049838fd76e86cf1cb90e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/4930f3368f2f48d98a9ab60fd6f354a1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/db4703b6047443b995d6d99d3d0a038e] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=36.7 K 2024-11-27T21:16:42,287 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:42,287 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/214220df3e084630b4254a9ea9a18857, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/6b6c3dc5e7a24671a7f350f7e285fd23, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5fca7d1127b944e98788e7af9b24badb] 2024-11-27T21:16:42,287 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fa15c839b5049838fd76e86cf1cb90e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732742197199 2024-11-27T21:16:42,287 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 214220df3e084630b4254a9ea9a18857, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732742197199 2024-11-27T21:16:42,287 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 4930f3368f2f48d98a9ab60fd6f354a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732742197891 2024-11-27T21:16:42,288 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting db4703b6047443b995d6d99d3d0a038e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732742200075 2024-11-27T21:16:42,288 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b6c3dc5e7a24671a7f350f7e285fd23, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732742197891 2024-11-27T21:16:42,288 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fca7d1127b944e98788e7af9b24badb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732742200073 2024-11-27T21:16:42,294 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#B#compaction#541 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:42,294 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/2c878b59978842d1ac58a704b4dfe589 is 50, key is test_row_0/B:col10/1732742200075/Put/seqid=0 2024-11-27T21:16:42,296 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:42,297 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127eae1d5297b494399924b46bef1efd33c_b6db78cfa2317a7fa812b5763434be10 store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:42,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742468_1644 (size=13051) 2024-11-27T21:16:42,299 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127eae1d5297b494399924b46bef1efd33c_b6db78cfa2317a7fa812b5763434be10, store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:42,299 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127eae1d5297b494399924b46bef1efd33c_b6db78cfa2317a7fa812b5763434be10 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:42,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742469_1645 (size=4469) 2024-11-27T21:16:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:42,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-27T21:16:42,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:42,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:42,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:42,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:42,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:42,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:42,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fc11b3f07dab4de480fc224990ffc60b_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742202265/Put/seqid=0 2024-11-27T21:16:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T21:16:42,389 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:42,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T21:16:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-27T21:16:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T21:16:42,392 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T21:16:42,393 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T21:16:42,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T21:16:42,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742262388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:42,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:42,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742470_1646 (size=14994) 2024-11-27T21:16:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742262390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:42,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742262390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:42,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:42,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742262394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T21:16:42,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742262499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:42,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742262499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:42,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742262499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:42,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742262508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:42,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T21:16:42,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:42,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:42,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T21:16:42,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:42,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:42,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:42,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:42,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:42,702 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/2c878b59978842d1ac58a704b4dfe589 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/2c878b59978842d1ac58a704b4dfe589 2024-11-27T21:16:42,705 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#A#compaction#542 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:42,706 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/2aec4c91f23945d0a009b31aca55d0d8 is 175, key is test_row_0/A:col10/1732742200075/Put/seqid=0 2024-11-27T21:16:42,708 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/B of b6db78cfa2317a7fa812b5763434be10 into 2c878b59978842d1ac58a704b4dfe589(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:42,708 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:42,708 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/B, priority=13, startTime=1732742202286; duration=0sec 2024-11-27T21:16:42,709 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:42,709 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:B 2024-11-27T21:16:42,709 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T21:16:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T21:16:42,710 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T21:16:42,710 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/C is initiating minor compaction (all files) 2024-11-27T21:16:42,710 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/C in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:42,710 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/7c34d29de0ce4a48b3bec22bbb7c0abd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/508cca7576b041b19f32e9dee13f62d8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/69208914247e4906abd3f7f5d6e48fe8] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=36.7 K 2024-11-27T21:16:42,711 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c34d29de0ce4a48b3bec22bbb7c0abd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732742197199 2024-11-27T21:16:42,711 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 508cca7576b041b19f32e9dee13f62d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732742197891 2024-11-27T21:16:42,711 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 69208914247e4906abd3f7f5d6e48fe8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732742200075 2024-11-27T21:16:42,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742471_1647 (size=32005) 2024-11-27T21:16:42,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742262710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742262710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742262710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:42,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:42,717 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#C#compaction#544 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:42,717 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/d4cdf79273be41fdb9956910ab6e5683 is 50, key is test_row_0/C:col10/1732742200075/Put/seqid=0 2024-11-27T21:16:42,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742472_1648 (size=13051) 2024-11-27T21:16:42,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742262719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:42,796 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:42,799 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fc11b3f07dab4de480fc224990ffc60b_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fc11b3f07dab4de480fc224990ffc60b_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:42,799 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/5107630494b24fd8a0f491e63d41414a, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:42,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/5107630494b24fd8a0f491e63d41414a is 175, key is test_row_0/A:col10/1732742202265/Put/seqid=0 2024-11-27T21:16:42,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742473_1649 (size=39949) 2024-11-27T21:16:42,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T21:16:42,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:42,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:42,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:42,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:42,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:43,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T21:16:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:43,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T21:16:43,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742263019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742263019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742263019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:43,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742263029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:43,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:43,117 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/2aec4c91f23945d0a009b31aca55d0d8 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/2aec4c91f23945d0a009b31aca55d0d8 2024-11-27T21:16:43,121 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/A of b6db78cfa2317a7fa812b5763434be10 into 2aec4c91f23945d0a009b31aca55d0d8(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:43,121 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:43,121 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/A, priority=13, startTime=1732742202286; duration=0sec 2024-11-27T21:16:43,121 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:43,121 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:A 2024-11-27T21:16:43,124 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/d4cdf79273be41fdb9956910ab6e5683 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/d4cdf79273be41fdb9956910ab6e5683 2024-11-27T21:16:43,128 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/C of b6db78cfa2317a7fa812b5763434be10 into d4cdf79273be41fdb9956910ab6e5683(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:43,128 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:43,128 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/C, priority=13, startTime=1732742202286; duration=0sec 2024-11-27T21:16:43,128 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:43,128 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:C 2024-11-27T21:16:43,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T21:16:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:43,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T21:16:43,204 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=445, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/5107630494b24fd8a0f491e63d41414a 2024-11-27T21:16:43,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/b9deb3e8df0c4b22b7f8feb745b49839 is 50, key is test_row_0/B:col10/1732742202265/Put/seqid=0 2024-11-27T21:16:43,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742474_1650 (size=12301) 2024-11-27T21:16:43,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/b9deb3e8df0c4b22b7f8feb745b49839 2024-11-27T21:16:43,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/1b2d37bcd00e47eca28e300acf72e94b is 50, key is test_row_0/C:col10/1732742202265/Put/seqid=0 2024-11-27T21:16:43,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742475_1651 (size=12301) 2024-11-27T21:16:43,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/1b2d37bcd00e47eca28e300acf72e94b 2024-11-27T21:16:43,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/5107630494b24fd8a0f491e63d41414a as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5107630494b24fd8a0f491e63d41414a 2024-11-27T21:16:43,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5107630494b24fd8a0f491e63d41414a, entries=200, sequenceid=445, filesize=39.0 K 2024-11-27T21:16:43,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/b9deb3e8df0c4b22b7f8feb745b49839 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/b9deb3e8df0c4b22b7f8feb745b49839 2024-11-27T21:16:43,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/b9deb3e8df0c4b22b7f8feb745b49839, entries=150, sequenceid=445, filesize=12.0 K 2024-11-27T21:16:43,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/1b2d37bcd00e47eca28e300acf72e94b as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/1b2d37bcd00e47eca28e300acf72e94b 2024-11-27T21:16:43,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/1b2d37bcd00e47eca28e300acf72e94b, entries=150, sequenceid=445, filesize=12.0 K 2024-11-27T21:16:43,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for b6db78cfa2317a7fa812b5763434be10 in 858ms, sequenceid=445, compaction requested=false 2024-11-27T21:16:43,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:43,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37381 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T21:16:43,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:43,310 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-27T21:16:43,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:43,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:43,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:43,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:43,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:43,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:43,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112731d4c9a740c1403684978e6822152d71_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742202387/Put/seqid=0 2024-11-27T21:16:43,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742476_1652 (size=12454) 2024-11-27T21:16:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T21:16:43,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:43,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. as already flushing 2024-11-27T21:16:43,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732742203549, max timestamp=1732742203559Num uniques 859; 2024-11-27T21:16:43,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-27T21:16:43,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732742203549, max timestamp=1732742203559Num uniques 859; 2024-11-27T21:16:43,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-27T21:16:43,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732742203549, max timestamp=1732742203559Num uniques 859; 2024-11-27T21:16:43,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4585c39bed03:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-27T21:16:43,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742263585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:43,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742263585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:43,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:43,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:43,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742263586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:43,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:43,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:43,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742263590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:43,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:43,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742263699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:43,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742263699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742263699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:43,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742263699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:43,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:43,725 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112731d4c9a740c1403684978e6822152d71_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112731d4c9a740c1403684978e6822152d71_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:43,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/5a00d2cf2c404f2193d18329b2d11dae, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:43,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/5a00d2cf2c404f2193d18329b2d11dae is 175, key is test_row_0/A:col10/1732742202387/Put/seqid=0 2024-11-27T21:16:43,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742477_1653 (size=31255) 2024-11-27T21:16:43,731 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=459, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/5a00d2cf2c404f2193d18329b2d11dae 2024-11-27T21:16:43,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/cd7e8b4b8e674c91887902ba8950101e is 50, key is test_row_0/B:col10/1732742202387/Put/seqid=0 2024-11-27T21:16:43,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742478_1654 (size=12301) 2024-11-27T21:16:43,744 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/cd7e8b4b8e674c91887902ba8950101e 2024-11-27T21:16:43,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/8376a39d5ff34dc89ad1f62225c96932 is 50, key is test_row_0/C:col10/1732742202387/Put/seqid=0 2024-11-27T21:16:43,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742479_1655 (size=12301) 2024-11-27T21:16:43,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742263909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:43,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742263909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:43,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:43,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:43,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742263910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742263910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:43,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,153 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/8376a39d5ff34dc89ad1f62225c96932 2024-11-27T21:16:44,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/5a00d2cf2c404f2193d18329b2d11dae as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5a00d2cf2c404f2193d18329b2d11dae 2024-11-27T21:16:44,160 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5a00d2cf2c404f2193d18329b2d11dae, entries=150, sequenceid=459, filesize=30.5 K 2024-11-27T21:16:44,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/cd7e8b4b8e674c91887902ba8950101e as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/cd7e8b4b8e674c91887902ba8950101e 2024-11-27T21:16:44,164 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/cd7e8b4b8e674c91887902ba8950101e, entries=150, sequenceid=459, filesize=12.0 K 2024-11-27T21:16:44,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/8376a39d5ff34dc89ad1f62225c96932 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8376a39d5ff34dc89ad1f62225c96932 2024-11-27T21:16:44,167 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8376a39d5ff34dc89ad1f62225c96932, entries=150, sequenceid=459, filesize=12.0 K 2024-11-27T21:16:44,168 INFO [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for b6db78cfa2317a7fa812b5763434be10 in 858ms, sequenceid=459, compaction requested=true 2024-11-27T21:16:44,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:44,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:44,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4585c39bed03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-27T21:16:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-27T21:16:44,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-27T21:16:44,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7770 sec 2024-11-27T21:16:44,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.7820 sec 2024-11-27T21:16:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:44,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-27T21:16:44,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:44,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:44,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:44,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:44,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:44,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T21:16:44,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127604ce981daac4ccbbf29fe6f0347952a_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742204222/Put/seqid=0 2024-11-27T21:16:44,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742480_1656 (size=14994) 2024-11-27T21:16:44,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742264233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742264234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742264234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742264236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34484 deadline: 1732742264279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,284 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4585c39bed03,37381,1732742022265 for 'test_row_1' in b6db78cfa2317a7fa812b5763434be10 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T21:16:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,326 INFO [Thread-2485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2485. 2024-11-27T21:16:44,326 DEBUG [Thread-2485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:44,326 DEBUG [Thread-2485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:44,326 DEBUG [Thread-2485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:44,326 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:44,327 INFO [Thread-2491 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2491. 2024-11-27T21:16:44,327 DEBUG [Thread-2491 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:44,327 DEBUG [Thread-2491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:44,327 DEBUG [Thread-2491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:44,328 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:44,330 INFO [Thread-2489 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2489. 2024-11-27T21:16:44,330 DEBUG [Thread-2489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:44,330 DEBUG [Thread-2489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:44,331 DEBUG [Thread-2489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:44,331 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:44,332 INFO [Thread-2483 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2483. 2024-11-27T21:16:44,332 DEBUG [Thread-2483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:44,332 DEBUG [Thread-2483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:44,332 DEBUG [Thread-2483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:44,332 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:44,333 INFO [Thread-2487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2487. 2024-11-27T21:16:44,333 DEBUG [Thread-2487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:44,333 DEBUG [Thread-2487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:44,333 DEBUG [Thread-2487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:44,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:44,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742264349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742264349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742264349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742264349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T21:16:44,549 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:44,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742264558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742264558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742264558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742264558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,635 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:44,638 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127604ce981daac4ccbbf29fe6f0347952a_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127604ce981daac4ccbbf29fe6f0347952a_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:44,638 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/ed5234cc8af940ad9d14e00603938188, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:44,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/ed5234cc8af940ad9d14e00603938188 is 175, key is test_row_0/A:col10/1732742204222/Put/seqid=0 2024-11-27T21:16:44,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742481_1657 (size=39949) 2024-11-27T21:16:44,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742264869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742264869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742264869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742264869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:45,042 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=494, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/ed5234cc8af940ad9d14e00603938188 2024-11-27T21:16:45,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/2dd89dc56eb84c3ba2207642f1de4ed7 is 50, key is test_row_0/B:col10/1732742204222/Put/seqid=0 2024-11-27T21:16:45,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742482_1658 (size=12301) 2024-11-27T21:16:45,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:45,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:45,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34526 deadline: 1732742265379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1732742265379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1732742265379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:45,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T21:16:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34472 deadline: 1732742265380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 2024-11-27T21:16:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 , the old value is region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T21:16:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., hostname=4585c39bed03,37381,1732742022265, seqNum=5 because the exception is null or not the one we care about 2024-11-27T21:16:45,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/2dd89dc56eb84c3ba2207642f1de4ed7 2024-11-27T21:16:45,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/8ba1eb2cfb4f4caf8baedacd06de82d1 is 50, key is test_row_0/C:col10/1732742204222/Put/seqid=0 2024-11-27T21:16:45,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742483_1659 (size=12301) 2024-11-27T21:16:45,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/8ba1eb2cfb4f4caf8baedacd06de82d1 2024-11-27T21:16:45,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/ed5234cc8af940ad9d14e00603938188 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/ed5234cc8af940ad9d14e00603938188 2024-11-27T21:16:45,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/ed5234cc8af940ad9d14e00603938188, entries=200, sequenceid=494, filesize=39.0 K 2024-11-27T21:16:45,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/2dd89dc56eb84c3ba2207642f1de4ed7 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/2dd89dc56eb84c3ba2207642f1de4ed7 2024-11-27T21:16:45,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/2dd89dc56eb84c3ba2207642f1de4ed7, entries=150, sequenceid=494, filesize=12.0 K 2024-11-27T21:16:45,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/8ba1eb2cfb4f4caf8baedacd06de82d1 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8ba1eb2cfb4f4caf8baedacd06de82d1 2024-11-27T21:16:45,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8ba1eb2cfb4f4caf8baedacd06de82d1, entries=150, sequenceid=494, filesize=12.0 K 2024-11-27T21:16:45,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for b6db78cfa2317a7fa812b5763434be10 in 1648ms, sequenceid=494, compaction requested=true 2024-11-27T21:16:45,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:45,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T21:16:45,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:45,872 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:45,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T21:16:45,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:45,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6db78cfa2317a7fa812b5763434be10:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T21:16:45,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:45,872 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:45,872 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:45,873 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/A is initiating minor compaction (all files) 2024-11-27T21:16:45,873 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:45,873 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/A in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:45,873 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/B is initiating minor compaction (all files) 2024-11-27T21:16:45,873 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/B in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:45,873 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/2aec4c91f23945d0a009b31aca55d0d8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5107630494b24fd8a0f491e63d41414a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5a00d2cf2c404f2193d18329b2d11dae, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/ed5234cc8af940ad9d14e00603938188] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=139.8 K 2024-11-27T21:16:45,873 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:45,873 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/2c878b59978842d1ac58a704b4dfe589, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/b9deb3e8df0c4b22b7f8feb745b49839, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/cd7e8b4b8e674c91887902ba8950101e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/2dd89dc56eb84c3ba2207642f1de4ed7] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=48.8 K 2024-11-27T21:16:45,873 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. files: [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/2aec4c91f23945d0a009b31aca55d0d8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5107630494b24fd8a0f491e63d41414a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5a00d2cf2c404f2193d18329b2d11dae, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/ed5234cc8af940ad9d14e00603938188] 2024-11-27T21:16:45,873 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c878b59978842d1ac58a704b4dfe589, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732742200075 2024-11-27T21:16:45,873 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2aec4c91f23945d0a009b31aca55d0d8, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732742200075 2024-11-27T21:16:45,873 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting b9deb3e8df0c4b22b7f8feb745b49839, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732742202265 2024-11-27T21:16:45,873 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5107630494b24fd8a0f491e63d41414a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732742202263 2024-11-27T21:16:45,873 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting cd7e8b4b8e674c91887902ba8950101e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732742202387 2024-11-27T21:16:45,873 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a00d2cf2c404f2193d18329b2d11dae, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732742202387 2024-11-27T21:16:45,874 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dd89dc56eb84c3ba2207642f1de4ed7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732742203576 2024-11-27T21:16:45,874 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed5234cc8af940ad9d14e00603938188, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732742203576 2024-11-27T21:16:45,880 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#B#compaction#553 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:45,880 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/cdc1dfb580684f8abca292aa9d1eb821 is 50, key is test_row_0/B:col10/1732742204222/Put/seqid=0 2024-11-27T21:16:45,883 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:45,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742484_1660 (size=13187) 2024-11-27T21:16:45,888 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127db43eb8758d947b4b19705314462b4eb_b6db78cfa2317a7fa812b5763434be10 store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:45,910 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127db43eb8758d947b4b19705314462b4eb_b6db78cfa2317a7fa812b5763434be10, store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:45,910 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127db43eb8758d947b4b19705314462b4eb_b6db78cfa2317a7fa812b5763434be10 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:45,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742485_1661 (size=4469) 2024-11-27T21:16:46,289 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/cdc1dfb580684f8abca292aa9d1eb821 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/cdc1dfb580684f8abca292aa9d1eb821 2024-11-27T21:16:46,292 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/B of b6db78cfa2317a7fa812b5763434be10 into cdc1dfb580684f8abca292aa9d1eb821(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:46,292 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:46,292 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/B, priority=12, startTime=1732742205872; duration=0sec 2024-11-27T21:16:46,292 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T21:16:46,292 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:B 2024-11-27T21:16:46,292 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T21:16:46,293 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T21:16:46,293 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1541): b6db78cfa2317a7fa812b5763434be10/C is initiating minor compaction (all files) 2024-11-27T21:16:46,293 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6db78cfa2317a7fa812b5763434be10/C in TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:46,293 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/d4cdf79273be41fdb9956910ab6e5683, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/1b2d37bcd00e47eca28e300acf72e94b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8376a39d5ff34dc89ad1f62225c96932, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8ba1eb2cfb4f4caf8baedacd06de82d1] into tmpdir=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp, totalSize=48.8 K 2024-11-27T21:16:46,293 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting d4cdf79273be41fdb9956910ab6e5683, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732742200075 2024-11-27T21:16:46,294 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b2d37bcd00e47eca28e300acf72e94b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732742202265 2024-11-27T21:16:46,294 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 8376a39d5ff34dc89ad1f62225c96932, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732742202387 2024-11-27T21:16:46,294 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ba1eb2cfb4f4caf8baedacd06de82d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732742203576 2024-11-27T21:16:46,300 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#C#compaction#555 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:46,301 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/1922081233f74f8eb3fb89a9eb2f79f3 is 50, key is test_row_0/C:col10/1732742204222/Put/seqid=0 2024-11-27T21:16:46,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742486_1662 (size=13187) 2024-11-27T21:16:46,314 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6db78cfa2317a7fa812b5763434be10#A#compaction#554 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T21:16:46,315 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/e2f4a25b871f4bcf8e04669b189eaf55 is 175, key is test_row_0/A:col10/1732742204222/Put/seqid=0 2024-11-27T21:16:46,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742487_1663 (size=32141) 2024-11-27T21:16:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37381 {}] regionserver.HRegion(8855): Flush requested on b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:46,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-27T21:16:46,391 INFO [Thread-2480 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2480. 2024-11-27T21:16:46,391 INFO [Thread-2474 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2474. 2024-11-27T21:16:46,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:46,391 DEBUG [Thread-2480 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:46,391 DEBUG [Thread-2474 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:46,391 INFO [Thread-2472 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2472. 2024-11-27T21:16:46,391 DEBUG [Thread-2480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:46,391 DEBUG [Thread-2474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:46,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:46,391 DEBUG [Thread-2472 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:46,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:46,391 DEBUG [Thread-2472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:46,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:46,391 DEBUG [Thread-2474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:46,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:46,391 DEBUG [Thread-2480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:46,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:46,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:46,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:46,391 DEBUG [Thread-2472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:46,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:46,393 INFO [Thread-2476 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2476. 2024-11-27T21:16:46,393 DEBUG [Thread-2476 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:46,393 DEBUG [Thread-2476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:46,393 DEBUG [Thread-2476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:46,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:46,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278a6794a82e2746cd857da6a38792812f_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_0/A:col10/1732742204233/Put/seqid=0 2024-11-27T21:16:46,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742488_1664 (size=12454) 2024-11-27T21:16:46,708 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/1922081233f74f8eb3fb89a9eb2f79f3 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/1922081233f74f8eb3fb89a9eb2f79f3 2024-11-27T21:16:46,712 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/C of b6db78cfa2317a7fa812b5763434be10 into 1922081233f74f8eb3fb89a9eb2f79f3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:46,712 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:46,712 INFO [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/C, priority=12, startTime=1732742205872; duration=0sec 2024-11-27T21:16:46,712 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:46,712 DEBUG [RS:0;4585c39bed03:37381-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:C 2024-11-27T21:16:46,722 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/e2f4a25b871f4bcf8e04669b189eaf55 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/e2f4a25b871f4bcf8e04669b189eaf55 2024-11-27T21:16:46,725 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6db78cfa2317a7fa812b5763434be10/A of b6db78cfa2317a7fa812b5763434be10 into e2f4a25b871f4bcf8e04669b189eaf55(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T21:16:46,725 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:46,725 INFO [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., storeName=b6db78cfa2317a7fa812b5763434be10/A, priority=12, startTime=1732742205872; duration=0sec 2024-11-27T21:16:46,725 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T21:16:46,725 DEBUG [RS:0;4585c39bed03:37381-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6db78cfa2317a7fa812b5763434be10:A 2024-11-27T21:16:46,801 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:46,805 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278a6794a82e2746cd857da6a38792812f_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278a6794a82e2746cd857da6a38792812f_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:46,806 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/19f189c7c1fd461f92fdcd54626c310f, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:46,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/19f189c7c1fd461f92fdcd54626c310f is 175, key is test_row_0/A:col10/1732742204233/Put/seqid=0 2024-11-27T21:16:46,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742489_1665 (size=31255) 2024-11-27T21:16:46,815 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=508, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/19f189c7c1fd461f92fdcd54626c310f 2024-11-27T21:16:46,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/6f28c09d6fbd477e87838e214cebc52f is 50, key is test_row_0/B:col10/1732742204233/Put/seqid=0 2024-11-27T21:16:46,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742490_1666 (size=12301) 2024-11-27T21:16:47,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/6f28c09d6fbd477e87838e214cebc52f 2024-11-27T21:16:47,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/149d18c7db514554b772f6302b7e7d50 is 50, key is test_row_0/C:col10/1732742204233/Put/seqid=0 2024-11-27T21:16:47,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742491_1667 (size=12301) 2024-11-27T21:16:47,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/149d18c7db514554b772f6302b7e7d50 2024-11-27T21:16:47,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/19f189c7c1fd461f92fdcd54626c310f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/19f189c7c1fd461f92fdcd54626c310f 2024-11-27T21:16:47,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/19f189c7c1fd461f92fdcd54626c310f, entries=150, sequenceid=508, filesize=30.5 K 2024-11-27T21:16:47,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/6f28c09d6fbd477e87838e214cebc52f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/6f28c09d6fbd477e87838e214cebc52f 2024-11-27T21:16:47,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/6f28c09d6fbd477e87838e214cebc52f, entries=150, sequenceid=508, filesize=12.0 K 2024-11-27T21:16:47,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/149d18c7db514554b772f6302b7e7d50 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/149d18c7db514554b772f6302b7e7d50 2024-11-27T21:16:47,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/149d18c7db514554b772f6302b7e7d50, entries=150, sequenceid=508, filesize=12.0 K 2024-11-27T21:16:47,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=6.71 KB/6870 for b6db78cfa2317a7fa812b5763434be10 in 867ms, sequenceid=508, compaction requested=false 2024-11-27T21:16:47,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6db78cfa2317a7fa812b5763434be10: 2024-11-27T21:16:54,341 INFO [Thread-2478 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2478. 2024-11-27T21:16:54,341 DEBUG [Thread-2478 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T21:16:54,341 DEBUG [Thread-2478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:54,341 DEBUG [Thread-2478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:54,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:54,341 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T21:16:54,341 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-27T21:16:54,341 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-27T21:16:54,341 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-27T21:16:54,341 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-27T21:16:54,341 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-27T21:16:54,341 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T21:16:54,341 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2841 2024-11-27T21:16:54,341 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2686 2024-11-27T21:16:54,341 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2848 2024-11-27T21:16:54,341 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2852 2024-11-27T21:16:54,341 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2690 2024-11-27T21:16:54,341 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T21:16:54,341 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T21:16:54,341 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:16:54,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:54,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:54,342 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:54,342 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T21:16:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T21:16:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T21:16:54,345 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742214344"}]},"ts":"1732742214344"} 2024-11-27T21:16:54,346 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T21:16:54,346 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T21:16:54,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T21:16:54,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6db78cfa2317a7fa812b5763434be10, UNASSIGN}] 2024-11-27T21:16:54,348 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6db78cfa2317a7fa812b5763434be10, UNASSIGN 2024-11-27T21:16:54,348 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=b6db78cfa2317a7fa812b5763434be10, regionState=CLOSING, regionLocation=4585c39bed03,37381,1732742022265 2024-11-27T21:16:54,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6db78cfa2317a7fa812b5763434be10, UNASSIGN because future has completed 2024-11-27T21:16:54,349 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T21:16:54,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265}] 2024-11-27T21:16:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T21:16:54,501 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:54,501 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T21:16:54,501 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing b6db78cfa2317a7fa812b5763434be10, disabling compactions & flushes 2024-11-27T21:16:54,501 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:54,501 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:54,501 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. after waiting 0 ms 2024-11-27T21:16:54,501 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:54,501 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing b6db78cfa2317a7fa812b5763434be10 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-27T21:16:54,502 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=A 2024-11-27T21:16:54,502 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:54,502 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=B 2024-11-27T21:16:54,502 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:54,502 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6db78cfa2317a7fa812b5763434be10, store=C 2024-11-27T21:16:54,502 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T21:16:54,506 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bc52ed2d81114496b3956887ed35631e_b6db78cfa2317a7fa812b5763434be10 is 50, key is test_row_1/A:col10/1732742214339/Put/seqid=0 2024-11-27T21:16:54,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742492_1668 (size=7374) 2024-11-27T21:16:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T21:16:54,909 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T21:16:54,912 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bc52ed2d81114496b3956887ed35631e_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bc52ed2d81114496b3956887ed35631e_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:54,913 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/f01b3776232046778dc08321777013fe, store: [table=TestAcidGuarantees family=A region=b6db78cfa2317a7fa812b5763434be10] 2024-11-27T21:16:54,913 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/f01b3776232046778dc08321777013fe is 175, key is test_row_1/A:col10/1732742214339/Put/seqid=0 2024-11-27T21:16:54,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742493_1669 (size=13865) 2024-11-27T21:16:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T21:16:55,317 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=515, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/f01b3776232046778dc08321777013fe 2024-11-27T21:16:55,322 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/e08ae287a1804053b732890dc14882be is 50, key is test_row_1/B:col10/1732742214339/Put/seqid=0 2024-11-27T21:16:55,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742494_1670 (size=7415) 2024-11-27T21:16:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T21:16:55,726 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/e08ae287a1804053b732890dc14882be 2024-11-27T21:16:55,731 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/cfd9240ce58d429da554c6ad3d45f3ed is 50, key is test_row_1/C:col10/1732742214339/Put/seqid=0 2024-11-27T21:16:55,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742495_1671 (size=7415) 2024-11-27T21:16:56,134 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/cfd9240ce58d429da554c6ad3d45f3ed 2024-11-27T21:16:56,138 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/A/f01b3776232046778dc08321777013fe as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/f01b3776232046778dc08321777013fe 2024-11-27T21:16:56,141 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/f01b3776232046778dc08321777013fe, entries=50, sequenceid=515, filesize=13.5 K 2024-11-27T21:16:56,141 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/B/e08ae287a1804053b732890dc14882be as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/e08ae287a1804053b732890dc14882be 2024-11-27T21:16:56,144 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/e08ae287a1804053b732890dc14882be, entries=50, sequenceid=515, filesize=7.2 K 2024-11-27T21:16:56,144 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/.tmp/C/cfd9240ce58d429da554c6ad3d45f3ed as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/cfd9240ce58d429da554c6ad3d45f3ed 2024-11-27T21:16:56,147 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/cfd9240ce58d429da554c6ad3d45f3ed, entries=50, sequenceid=515, filesize=7.2 K 2024-11-27T21:16:56,148 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for b6db78cfa2317a7fa812b5763434be10 in 1646ms, sequenceid=515, compaction requested=true 2024-11-27T21:16:56,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/1e3c5263d70748e5a7743eacf3c33994, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/e35da90759984e349961dbb671e977a2, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/b79affaa7ded485c827235234ecf1dba, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/4f853468abf3420887ea9041f8e2c78e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/7d3a39d6415a44d2a557fc7d3d592514, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/f8882c37abf349cc900c3597e323f504, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/fbf328acbc6e44bbacd2719896b999c9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/3690bd1f91eb4a39b5f9f01ef9db957c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/307d9e124a5c4d3d8af89827607aeec0, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/358de47f83674e97919957c278508ceb, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/9fe5ff8755cb47d3a19a583f78b66b1d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/fbdb360f1a224ce29b2456e8562fdd53, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/d27a2d4db52b45c49c4319152246780b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/3b2fbfa0f2a54c43a68573e10ffa0781, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/e22fdf004a704ba8acb59a3320be618f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/47df32267bcb45ba9654b03d18ce57c1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/259b07cbf3774634acd0529944b5e7e9, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5a9ff3f868ad408e857709d04b2def55, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/214220df3e084630b4254a9ea9a18857, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/96777d3c2fc54e028c3071dd3c49514b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/6b6c3dc5e7a24671a7f350f7e285fd23, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5fca7d1127b944e98788e7af9b24badb, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/2aec4c91f23945d0a009b31aca55d0d8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5107630494b24fd8a0f491e63d41414a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5a00d2cf2c404f2193d18329b2d11dae, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/ed5234cc8af940ad9d14e00603938188] to archive 2024-11-27T21:16:56,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:16:56,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/1e3c5263d70748e5a7743eacf3c33994 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/1e3c5263d70748e5a7743eacf3c33994 2024-11-27T21:16:56,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/e35da90759984e349961dbb671e977a2 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/e35da90759984e349961dbb671e977a2 2024-11-27T21:16:56,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/b79affaa7ded485c827235234ecf1dba to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/b79affaa7ded485c827235234ecf1dba 2024-11-27T21:16:56,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/4f853468abf3420887ea9041f8e2c78e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/4f853468abf3420887ea9041f8e2c78e 2024-11-27T21:16:56,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/7d3a39d6415a44d2a557fc7d3d592514 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/7d3a39d6415a44d2a557fc7d3d592514 2024-11-27T21:16:56,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/f8882c37abf349cc900c3597e323f504 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/f8882c37abf349cc900c3597e323f504 2024-11-27T21:16:56,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/fbf328acbc6e44bbacd2719896b999c9 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/fbf328acbc6e44bbacd2719896b999c9 2024-11-27T21:16:56,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/3690bd1f91eb4a39b5f9f01ef9db957c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/3690bd1f91eb4a39b5f9f01ef9db957c 2024-11-27T21:16:56,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/307d9e124a5c4d3d8af89827607aeec0 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/307d9e124a5c4d3d8af89827607aeec0 2024-11-27T21:16:56,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/358de47f83674e97919957c278508ceb to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/358de47f83674e97919957c278508ceb 2024-11-27T21:16:56,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/9fe5ff8755cb47d3a19a583f78b66b1d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/9fe5ff8755cb47d3a19a583f78b66b1d 2024-11-27T21:16:56,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/fbdb360f1a224ce29b2456e8562fdd53 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/fbdb360f1a224ce29b2456e8562fdd53 2024-11-27T21:16:56,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/d27a2d4db52b45c49c4319152246780b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/d27a2d4db52b45c49c4319152246780b 2024-11-27T21:16:56,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/3b2fbfa0f2a54c43a68573e10ffa0781 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/3b2fbfa0f2a54c43a68573e10ffa0781 2024-11-27T21:16:56,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/e22fdf004a704ba8acb59a3320be618f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/e22fdf004a704ba8acb59a3320be618f 2024-11-27T21:16:56,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/47df32267bcb45ba9654b03d18ce57c1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/47df32267bcb45ba9654b03d18ce57c1 2024-11-27T21:16:56,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/259b07cbf3774634acd0529944b5e7e9 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/259b07cbf3774634acd0529944b5e7e9 2024-11-27T21:16:56,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5a9ff3f868ad408e857709d04b2def55 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5a9ff3f868ad408e857709d04b2def55 2024-11-27T21:16:56,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/214220df3e084630b4254a9ea9a18857 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/214220df3e084630b4254a9ea9a18857 2024-11-27T21:16:56,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/96777d3c2fc54e028c3071dd3c49514b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/96777d3c2fc54e028c3071dd3c49514b 2024-11-27T21:16:56,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/6b6c3dc5e7a24671a7f350f7e285fd23 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/6b6c3dc5e7a24671a7f350f7e285fd23 2024-11-27T21:16:56,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5fca7d1127b944e98788e7af9b24badb to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5fca7d1127b944e98788e7af9b24badb 2024-11-27T21:16:56,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/2aec4c91f23945d0a009b31aca55d0d8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/2aec4c91f23945d0a009b31aca55d0d8 2024-11-27T21:16:56,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5107630494b24fd8a0f491e63d41414a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5107630494b24fd8a0f491e63d41414a 2024-11-27T21:16:56,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5a00d2cf2c404f2193d18329b2d11dae to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/5a00d2cf2c404f2193d18329b2d11dae 2024-11-27T21:16:56,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/ed5234cc8af940ad9d14e00603938188 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/ed5234cc8af940ad9d14e00603938188 2024-11-27T21:16:56,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/e5f15a80dfcc4311addc28fee9d1cee4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/90b90cbc7fd24b78bb2425a7c6761fef, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/462689a9b6194a7a8a4e19f149a037de, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/a88d7bb8a60149a39e76ad4d203644d5, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/c82c21e1817948a3993f8ca6c4ef9f9c, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/1614ad86f1e04d509f32e9842fb29914, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d1368ac5933148bba9ce015918e9810e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/c35074bb554a4e76bf3e2e454f0270a7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/07cf68e3551442fda3cfae5f732de2c4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d48a73c1134e4be381e95f47ee98c6b8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/6c4cd7e7ab444bc883cf1aa7fae59b71, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/bb5c7f269a23417aa5d188fa9dfa53fc, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d66ade8713f44b409136be42b3f70ef4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/5bb452c397d742a18eec4e9966d2b2bf, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/0ed5cd95234749c59379a07d28a45ec4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/0721ba5b5c4f4c41ab0e8b431c4234f6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/b4f333acf84244dd9084159d5eb17ffa, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/4473c2748f874460a64808961f06b63f, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/8fa15c839b5049838fd76e86cf1cb90e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/a617c6dfe81e4246b1b4cb4218c313d6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/4930f3368f2f48d98a9ab60fd6f354a1, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/2c878b59978842d1ac58a704b4dfe589, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/db4703b6047443b995d6d99d3d0a038e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/b9deb3e8df0c4b22b7f8feb745b49839, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/cd7e8b4b8e674c91887902ba8950101e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/2dd89dc56eb84c3ba2207642f1de4ed7] to archive 2024-11-27T21:16:56,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:16:56,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/e5f15a80dfcc4311addc28fee9d1cee4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/e5f15a80dfcc4311addc28fee9d1cee4 2024-11-27T21:16:56,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/90b90cbc7fd24b78bb2425a7c6761fef to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/90b90cbc7fd24b78bb2425a7c6761fef 2024-11-27T21:16:56,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/462689a9b6194a7a8a4e19f149a037de to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/462689a9b6194a7a8a4e19f149a037de 2024-11-27T21:16:56,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/a88d7bb8a60149a39e76ad4d203644d5 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/a88d7bb8a60149a39e76ad4d203644d5 2024-11-27T21:16:56,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/c82c21e1817948a3993f8ca6c4ef9f9c to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/c82c21e1817948a3993f8ca6c4ef9f9c 2024-11-27T21:16:56,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/1614ad86f1e04d509f32e9842fb29914 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/1614ad86f1e04d509f32e9842fb29914 2024-11-27T21:16:56,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d1368ac5933148bba9ce015918e9810e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d1368ac5933148bba9ce015918e9810e 2024-11-27T21:16:56,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/c35074bb554a4e76bf3e2e454f0270a7 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/c35074bb554a4e76bf3e2e454f0270a7 2024-11-27T21:16:56,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/07cf68e3551442fda3cfae5f732de2c4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/07cf68e3551442fda3cfae5f732de2c4 2024-11-27T21:16:56,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d48a73c1134e4be381e95f47ee98c6b8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d48a73c1134e4be381e95f47ee98c6b8 2024-11-27T21:16:56,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/6c4cd7e7ab444bc883cf1aa7fae59b71 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/6c4cd7e7ab444bc883cf1aa7fae59b71 2024-11-27T21:16:56,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/bb5c7f269a23417aa5d188fa9dfa53fc to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/bb5c7f269a23417aa5d188fa9dfa53fc 2024-11-27T21:16:56,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d66ade8713f44b409136be42b3f70ef4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/d66ade8713f44b409136be42b3f70ef4 2024-11-27T21:16:56,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/5bb452c397d742a18eec4e9966d2b2bf to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/5bb452c397d742a18eec4e9966d2b2bf 2024-11-27T21:16:56,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/0ed5cd95234749c59379a07d28a45ec4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/0ed5cd95234749c59379a07d28a45ec4 2024-11-27T21:16:56,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/0721ba5b5c4f4c41ab0e8b431c4234f6 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/0721ba5b5c4f4c41ab0e8b431c4234f6 2024-11-27T21:16:56,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/b4f333acf84244dd9084159d5eb17ffa to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/b4f333acf84244dd9084159d5eb17ffa 2024-11-27T21:16:56,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/4473c2748f874460a64808961f06b63f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/4473c2748f874460a64808961f06b63f 2024-11-27T21:16:56,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/8fa15c839b5049838fd76e86cf1cb90e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/8fa15c839b5049838fd76e86cf1cb90e 2024-11-27T21:16:56,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/a617c6dfe81e4246b1b4cb4218c313d6 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/a617c6dfe81e4246b1b4cb4218c313d6 2024-11-27T21:16:56,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/4930f3368f2f48d98a9ab60fd6f354a1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/4930f3368f2f48d98a9ab60fd6f354a1 2024-11-27T21:16:56,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/2c878b59978842d1ac58a704b4dfe589 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/2c878b59978842d1ac58a704b4dfe589 2024-11-27T21:16:56,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/db4703b6047443b995d6d99d3d0a038e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/db4703b6047443b995d6d99d3d0a038e 2024-11-27T21:16:56,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/b9deb3e8df0c4b22b7f8feb745b49839 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/b9deb3e8df0c4b22b7f8feb745b49839 2024-11-27T21:16:56,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/cd7e8b4b8e674c91887902ba8950101e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/cd7e8b4b8e674c91887902ba8950101e 2024-11-27T21:16:56,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/2dd89dc56eb84c3ba2207642f1de4ed7 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/2dd89dc56eb84c3ba2207642f1de4ed7 2024-11-27T21:16:56,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/f031a49e63ee447c8b7b09fc763d949a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/617fe78eb1dc4c17b1afeef9aa90ad9b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/2c8fe9aaffb04b0a84b13d0c3259ba8a, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8afb87bbc72d46338fb25a5c42622fcf, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/4c37337cfb8d406c9d18632588227697, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/cfb340ba66a647c6b5a880ffc0dd06d3, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/6eee38d0ab664f1bbeda108fd9da9a5d, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/1e3cebab6499407ca3877b2af6f91ed7, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/fbc3c9bb47d34ad98f7289a5cdc77958, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/da4b6f1e7be4425797634a188e2addc6, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/cbb9a5e1d88a45759f2bc9c88ab34059, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/e3779042807c4249b7b2ff33c341b149, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/6662ff5c10d84e9db5ae78976c7d46cf, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/507e7652d85c4ef5a7be69fb039fbf7e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8dc3e78316834ad2aab6f60c8593ebe3, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/c2e2f9375d84460f96eafd9bdfbd067e, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/5f2fb194f15a445b8d6d7ec2b6a5cd17, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/d39d92f1d9474e188e1ddbcd623dbdcf, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/7c34d29de0ce4a48b3bec22bbb7c0abd, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/ae3e7d8e309d402dbb47bd426fe01af4, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/508cca7576b041b19f32e9dee13f62d8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/d4cdf79273be41fdb9956910ab6e5683, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/69208914247e4906abd3f7f5d6e48fe8, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/1b2d37bcd00e47eca28e300acf72e94b, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8376a39d5ff34dc89ad1f62225c96932, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8ba1eb2cfb4f4caf8baedacd06de82d1] to archive 2024-11-27T21:16:56,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T21:16:56,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/f031a49e63ee447c8b7b09fc763d949a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/f031a49e63ee447c8b7b09fc763d949a 2024-11-27T21:16:56,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/617fe78eb1dc4c17b1afeef9aa90ad9b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/617fe78eb1dc4c17b1afeef9aa90ad9b 2024-11-27T21:16:56,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/2c8fe9aaffb04b0a84b13d0c3259ba8a to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/2c8fe9aaffb04b0a84b13d0c3259ba8a 2024-11-27T21:16:56,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8afb87bbc72d46338fb25a5c42622fcf to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8afb87bbc72d46338fb25a5c42622fcf 2024-11-27T21:16:56,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/4c37337cfb8d406c9d18632588227697 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/4c37337cfb8d406c9d18632588227697 2024-11-27T21:16:56,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/cfb340ba66a647c6b5a880ffc0dd06d3 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/cfb340ba66a647c6b5a880ffc0dd06d3 2024-11-27T21:16:56,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/6eee38d0ab664f1bbeda108fd9da9a5d to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/6eee38d0ab664f1bbeda108fd9da9a5d 2024-11-27T21:16:56,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/1e3cebab6499407ca3877b2af6f91ed7 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/1e3cebab6499407ca3877b2af6f91ed7 2024-11-27T21:16:56,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/fbc3c9bb47d34ad98f7289a5cdc77958 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/fbc3c9bb47d34ad98f7289a5cdc77958 2024-11-27T21:16:56,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/da4b6f1e7be4425797634a188e2addc6 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/da4b6f1e7be4425797634a188e2addc6 2024-11-27T21:16:56,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/cbb9a5e1d88a45759f2bc9c88ab34059 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/cbb9a5e1d88a45759f2bc9c88ab34059 2024-11-27T21:16:56,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/e3779042807c4249b7b2ff33c341b149 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/e3779042807c4249b7b2ff33c341b149 2024-11-27T21:16:56,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/6662ff5c10d84e9db5ae78976c7d46cf to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/6662ff5c10d84e9db5ae78976c7d46cf 2024-11-27T21:16:56,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/507e7652d85c4ef5a7be69fb039fbf7e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/507e7652d85c4ef5a7be69fb039fbf7e 2024-11-27T21:16:56,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8dc3e78316834ad2aab6f60c8593ebe3 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8dc3e78316834ad2aab6f60c8593ebe3 2024-11-27T21:16:56,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/c2e2f9375d84460f96eafd9bdfbd067e to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/c2e2f9375d84460f96eafd9bdfbd067e 2024-11-27T21:16:56,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/5f2fb194f15a445b8d6d7ec2b6a5cd17 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/5f2fb194f15a445b8d6d7ec2b6a5cd17 2024-11-27T21:16:56,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/d39d92f1d9474e188e1ddbcd623dbdcf to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/d39d92f1d9474e188e1ddbcd623dbdcf 2024-11-27T21:16:56,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/7c34d29de0ce4a48b3bec22bbb7c0abd to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/7c34d29de0ce4a48b3bec22bbb7c0abd 2024-11-27T21:16:56,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/ae3e7d8e309d402dbb47bd426fe01af4 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/ae3e7d8e309d402dbb47bd426fe01af4 2024-11-27T21:16:56,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/508cca7576b041b19f32e9dee13f62d8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/508cca7576b041b19f32e9dee13f62d8 2024-11-27T21:16:56,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/d4cdf79273be41fdb9956910ab6e5683 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/d4cdf79273be41fdb9956910ab6e5683 2024-11-27T21:16:56,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/69208914247e4906abd3f7f5d6e48fe8 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/69208914247e4906abd3f7f5d6e48fe8 2024-11-27T21:16:56,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/1b2d37bcd00e47eca28e300acf72e94b to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/1b2d37bcd00e47eca28e300acf72e94b 2024-11-27T21:16:56,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8376a39d5ff34dc89ad1f62225c96932 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8376a39d5ff34dc89ad1f62225c96932 2024-11-27T21:16:56,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8ba1eb2cfb4f4caf8baedacd06de82d1 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/8ba1eb2cfb4f4caf8baedacd06de82d1 2024-11-27T21:16:56,215 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/recovered.edits/518.seqid, newMaxSeqId=518, maxSeqId=4 2024-11-27T21:16:56,215 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. 2024-11-27T21:16:56,215 DEBUG [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for b6db78cfa2317a7fa812b5763434be10: Waiting for close lock at 1732742214501Running coprocessor pre-close hooks at 1732742214501Disabling compacts and flushes for region at 1732742214501Disabling writes for close at 1732742214501Obtaining lock to block concurrent updates at 1732742214501Preparing flush snapshotting stores in b6db78cfa2317a7fa812b5763434be10 at 1732742214501Finished memstore snapshotting TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732742214502 (+1 ms)Flushing stores of TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10. at 1732742214502Flushing b6db78cfa2317a7fa812b5763434be10/A: creating writer at 1732742214502Flushing mob file b6db78cfa2317a7fa812b5763434be10/A: appending metadata at 1732742214506 (+4 ms)Flushing mob file b6db78cfa2317a7fa812b5763434be10/A: closing flushed file at 1732742214506Flushing b6db78cfa2317a7fa812b5763434be10/A: appending metadata at 1732742214913 (+407 ms)Flushing b6db78cfa2317a7fa812b5763434be10/A: closing flushed file at 1732742214913Flushing b6db78cfa2317a7fa812b5763434be10/B: creating writer at 1732742215320 (+407 ms)Flushing b6db78cfa2317a7fa812b5763434be10/B: appending metadata at 1732742215322 (+2 ms)Flushing b6db78cfa2317a7fa812b5763434be10/B: closing flushed file at 1732742215322Flushing b6db78cfa2317a7fa812b5763434be10/C: creating writer at 1732742215728 (+406 ms)Flushing b6db78cfa2317a7fa812b5763434be10/C: appending metadata at 1732742215730 (+2 ms)Flushing b6db78cfa2317a7fa812b5763434be10/C: closing flushed file at 1732742215730Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f40cb0e: reopening flushed file at 1732742216137 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f4fb29a: reopening flushed file at 1732742216141 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@621a676f: reopening flushed file at 1732742216144 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for b6db78cfa2317a7fa812b5763434be10 in 1646ms, sequenceid=515, compaction requested=true at 1732742216148 (+4 ms)Writing region close event to WAL at 1732742216212 (+64 ms)Running coprocessor post-close hooks at 1732742216215 (+3 ms)Closed at 1732742216215 2024-11-27T21:16:56,217 INFO [RS_CLOSE_REGION-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,217 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=b6db78cfa2317a7fa812b5763434be10, regionState=CLOSED 2024-11-27T21:16:56,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 because future has completed 2024-11-27T21:16:56,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-27T21:16:56,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure b6db78cfa2317a7fa812b5763434be10, server=4585c39bed03,37381,1732742022265 in 1.8700 sec 2024-11-27T21:16:56,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-11-27T21:16:56,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6db78cfa2317a7fa812b5763434be10, UNASSIGN in 1.8730 sec 2024-11-27T21:16:56,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-27T21:16:56,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8760 sec 2024-11-27T21:16:56,224 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732742216224"}]},"ts":"1732742216224"} 2024-11-27T21:16:56,225 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T21:16:56,225 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T21:16:56,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8840 sec 2024-11-27T21:16:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T21:16:56,489 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:56,489 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T21:16:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:16:56,491 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:16:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-27T21:16:56,491 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:16:56,494 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,496 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C, FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/recovered.edits] 2024-11-27T21:16:56,497 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/19f189c7c1fd461f92fdcd54626c310f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/19f189c7c1fd461f92fdcd54626c310f 2024-11-27T21:16:56,498 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/e2f4a25b871f4bcf8e04669b189eaf55 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/e2f4a25b871f4bcf8e04669b189eaf55 2024-11-27T21:16:56,499 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/f01b3776232046778dc08321777013fe to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/A/f01b3776232046778dc08321777013fe 2024-11-27T21:16:56,501 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/6f28c09d6fbd477e87838e214cebc52f to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/6f28c09d6fbd477e87838e214cebc52f 2024-11-27T21:16:56,502 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/cdc1dfb580684f8abca292aa9d1eb821 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/cdc1dfb580684f8abca292aa9d1eb821 2024-11-27T21:16:56,502 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/e08ae287a1804053b732890dc14882be to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/B/e08ae287a1804053b732890dc14882be 2024-11-27T21:16:56,504 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/149d18c7db514554b772f6302b7e7d50 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/149d18c7db514554b772f6302b7e7d50 2024-11-27T21:16:56,505 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/1922081233f74f8eb3fb89a9eb2f79f3 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/1922081233f74f8eb3fb89a9eb2f79f3 2024-11-27T21:16:56,505 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/cfd9240ce58d429da554c6ad3d45f3ed to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/C/cfd9240ce58d429da554c6ad3d45f3ed 2024-11-27T21:16:56,507 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/recovered.edits/518.seqid to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10/recovered.edits/518.seqid 2024-11-27T21:16:56,507 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/default/TestAcidGuarantees/b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,508 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T21:16:56,508 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T21:16:56,509 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T21:16:56,511 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270072d20fbb7440fe99d7d495f8e3595c_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270072d20fbb7440fe99d7d495f8e3595c_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,511 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112707f4ca24e60549a69b42c5992dbb971b_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112707f4ca24e60549a69b42c5992dbb971b_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,512 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127090b98885918475b8defc0cfa49b7cfb_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127090b98885918475b8defc0cfa49b7cfb_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,513 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272c06762195134b2fa46538f5597c4b9a_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272c06762195134b2fa46538f5597c4b9a_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,514 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112731d4c9a740c1403684978e6822152d71_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112731d4c9a740c1403684978e6822152d71_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,515 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112739d53ac69a774bcc85dce2c7657b6548_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112739d53ac69a774bcc85dce2c7657b6548_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,516 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273de076dc2bc84b9fbcd22117e645464d_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273de076dc2bc84b9fbcd22117e645464d_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,517 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112740e57c0364484bd1b01629f0f6e69517_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112740e57c0364484bd1b01629f0f6e69517_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,518 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127604ce981daac4ccbbf29fe6f0347952a_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127604ce981daac4ccbbf29fe6f0347952a_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,518 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276b01410a681d4055a791e96a62bf0c7a_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276b01410a681d4055a791e96a62bf0c7a_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,519 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112770fba8acf9c14c958995b1d7e4340899_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112770fba8acf9c14c958995b1d7e4340899_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,520 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112772929aca399c496c8e905a9e2149ce53_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112772929aca399c496c8e905a9e2149ce53_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,521 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112787bbcc0d99ea4e0ab16d4c3e762cb0f5_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112787bbcc0d99ea4e0ab16d4c3e762cb0f5_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,521 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278a6794a82e2746cd857da6a38792812f_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278a6794a82e2746cd857da6a38792812f_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,522 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112799408590a432469ba439e6876e70bf78_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112799408590a432469ba439e6876e70bf78_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,523 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a6532e323ff14f37a4147b905f2517e9_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a6532e323ff14f37a4147b905f2517e9_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,523 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127abb9e1d15cb442ea8a80eebba8fec747_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127abb9e1d15cb442ea8a80eebba8fec747_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,524 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bc52ed2d81114496b3956887ed35631e_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bc52ed2d81114496b3956887ed35631e_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,525 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cd3e9786bdf54bd2b478dfb1c6b5db84_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cd3e9786bdf54bd2b478dfb1c6b5db84_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,526 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fc11b3f07dab4de480fc224990ffc60b_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fc11b3f07dab4de480fc224990ffc60b_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,526 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fc476de89499427aacdd358217406eee_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fc476de89499427aacdd358217406eee_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,527 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fdc069a933c145b59c9316eca2ec98c4_b6db78cfa2317a7fa812b5763434be10 to hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fdc069a933c145b59c9316eca2ec98c4_b6db78cfa2317a7fa812b5763434be10 2024-11-27T21:16:56,527 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T21:16:56,529 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:16:56,531 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T21:16:56,533 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T21:16:56,534 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:16:56,534 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T21:16:56,534 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732742216534"}]},"ts":"9223372036854775807"} 2024-11-27T21:16:56,536 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T21:16:56,536 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b6db78cfa2317a7fa812b5763434be10, NAME => 'TestAcidGuarantees,,1732742181229.b6db78cfa2317a7fa812b5763434be10.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T21:16:56,536 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T21:16:56,536 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732742216536"}]},"ts":"9223372036854775807"} 2024-11-27T21:16:56,537 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T21:16:56,538 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T21:16:56,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-27T21:16:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-27T21:16:56,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T21:16:56,600 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T21:16:56,609 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 251), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=290 (was 346), ProcessCount=11 (was 11), AvailableMemoryMB=7838 (was 7932) 2024-11-27T21:16:56,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T21:16:56,609 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T21:16:56,609 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:16:56,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:56,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:56,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T21:16:56,609 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T21:16:56,609 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=261653369, stopped=false 2024-11-27T21:16:56,610 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4585c39bed03,38503,1732742021440 2024-11-27T21:16:56,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T21:16:56,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T21:16:56,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:16:56,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:16:56,611 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T21:16:56,612 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T21:16:56,612 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T21:16:56,612 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T21:16:56,612 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:16:56,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:56,613 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4585c39bed03,37381,1732742022265' ***** 2024-11-27T21:16:56,613 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T21:16:56,613 INFO [RS:0;4585c39bed03:37381 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T21:16:56,614 INFO [RS:0;4585c39bed03:37381 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T21:16:56,614 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T21:16:56,614 INFO [RS:0;4585c39bed03:37381 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T21:16:56,614 INFO [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(959): stopping server 4585c39bed03,37381,1732742022265 2024-11-27T21:16:56,614 INFO [RS:0;4585c39bed03:37381 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T21:16:56,614 INFO [RS:0;4585c39bed03:37381 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4585c39bed03:37381. 2024-11-27T21:16:56,614 DEBUG [RS:0;4585c39bed03:37381 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T21:16:56,614 DEBUG [RS:0;4585c39bed03:37381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:56,614 INFO [RS:0;4585c39bed03:37381 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T21:16:56,614 INFO [RS:0;4585c39bed03:37381 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T21:16:56,615 INFO [RS:0;4585c39bed03:37381 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T21:16:56,615 INFO [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T21:16:56,615 INFO [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T21:16:56,615 DEBUG [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-27T21:16:56,616 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T21:16:56,616 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T21:16:56,616 DEBUG [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T21:16:56,616 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T21:16:56,616 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T21:16:56,616 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T21:16:56,616 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-27T21:16:56,640 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/.tmp/info/75da8926db9648f692cdda8bef249aa3 is 95, key is TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332./info:/1732742055722/DeleteFamily/seqid=0 2024-11-27T21:16:56,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742496_1672 (size=6514) 2024-11-27T21:16:56,662 INFO [regionserver/4585c39bed03:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T21:16:56,816 DEBUG [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T21:16:57,016 DEBUG [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T21:16:57,044 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/.tmp/info/75da8926db9648f692cdda8bef249aa3 2024-11-27T21:16:57,062 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/.tmp/ns/0185463e20734991b6bd55f48518d2ce is 93, key is TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332./ns:/1732742055677/DeleteFamily/seqid=0 2024-11-27T21:16:57,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742497_1673 (size=5906) 2024-11-27T21:16:57,216 DEBUG [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T21:16:57,278 INFO [regionserver/4585c39bed03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T21:16:57,278 INFO [regionserver/4585c39bed03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T21:16:57,416 DEBUG [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T21:16:57,466 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/.tmp/ns/0185463e20734991b6bd55f48518d2ce 2024-11-27T21:16:57,483 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/.tmp/rep_barrier/5f6ae22853ad4d8b8fed13cc57e559ce is 102, key is TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332./rep_barrier:/1732742055677/DeleteFamily/seqid=0 2024-11-27T21:16:57,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742498_1674 (size=6025) 2024-11-27T21:16:57,617 DEBUG [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T21:16:57,817 DEBUG [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T21:16:57,887 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/.tmp/rep_barrier/5f6ae22853ad4d8b8fed13cc57e559ce 2024-11-27T21:16:57,904 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/.tmp/table/a099abd7324045e0986751ea5585c97f is 96, key is TestAcidGuarantees,,1732742026373.cc764913a0f35f9f71e6ee97aeccf332./table:/1732742055677/DeleteFamily/seqid=0 2024-11-27T21:16:57,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742499_1675 (size=5936) 2024-11-27T21:16:58,017 DEBUG [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T21:16:58,217 DEBUG [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T21:16:58,307 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/.tmp/table/a099abd7324045e0986751ea5585c97f 2024-11-27T21:16:58,311 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/.tmp/info/75da8926db9648f692cdda8bef249aa3 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/info/75da8926db9648f692cdda8bef249aa3 2024-11-27T21:16:58,313 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/info/75da8926db9648f692cdda8bef249aa3, entries=12, sequenceid=90, filesize=6.4 K 2024-11-27T21:16:58,314 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/.tmp/ns/0185463e20734991b6bd55f48518d2ce as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/ns/0185463e20734991b6bd55f48518d2ce 2024-11-27T21:16:58,316 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/ns/0185463e20734991b6bd55f48518d2ce, entries=8, sequenceid=90, filesize=5.8 K 2024-11-27T21:16:58,317 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/.tmp/rep_barrier/5f6ae22853ad4d8b8fed13cc57e559ce as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/rep_barrier/5f6ae22853ad4d8b8fed13cc57e559ce 2024-11-27T21:16:58,319 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/rep_barrier/5f6ae22853ad4d8b8fed13cc57e559ce, entries=6, sequenceid=90, filesize=5.9 K 2024-11-27T21:16:58,320 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/.tmp/table/a099abd7324045e0986751ea5585c97f as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/table/a099abd7324045e0986751ea5585c97f 2024-11-27T21:16:58,322 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/table/a099abd7324045e0986751ea5585c97f, entries=7, sequenceid=90, filesize=5.8 K 2024-11-27T21:16:58,323 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1707ms, sequenceid=90, compaction requested=false 2024-11-27T21:16:58,326 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-27T21:16:58,327 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T21:16:58,327 INFO [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T21:16:58,327 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732742216615Running coprocessor pre-close hooks at 1732742216615Disabling compacts and flushes for region at 1732742216615Disabling writes for close at 1732742216616 (+1 ms)Obtaining lock to block concurrent updates at 1732742216616Preparing flush snapshotting stores in 1588230740 at 1732742216616Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732742216616Flushing stores of hbase:meta,,1.1588230740 at 1732742216617 (+1 ms)Flushing 1588230740/info: creating writer at 1732742216617Flushing 1588230740/info: appending metadata at 1732742216640 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732742216640Flushing 1588230740/ns: creating writer at 1732742217047 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732742217062 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732742217062Flushing 1588230740/rep_barrier: creating writer at 1732742217469 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732742217483 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732742217483Flushing 1588230740/table: creating writer at 1732742217890 (+407 ms)Flushing 1588230740/table: appending metadata at 1732742217904 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732742217904Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49fa22de: reopening flushed file at 1732742218310 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2397f892: reopening flushed file at 1732742218313 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1753d9c2: reopening flushed file at 1732742218316 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75223a7b: reopening flushed file at 1732742218319 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1707ms, sequenceid=90, compaction requested=false at 1732742218323 (+4 ms)Writing region close event to WAL at 1732742218324 (+1 ms)Running coprocessor post-close hooks at 1732742218327 (+3 ms)Closed at 1732742218327 2024-11-27T21:16:58,327 DEBUG [RS_CLOSE_META-regionserver/4585c39bed03:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T21:16:58,417 INFO [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(976): stopping server 4585c39bed03,37381,1732742022265; all regions closed. 2024-11-27T21:16:58,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741834_1010 (size=25344) 2024-11-27T21:16:58,423 DEBUG [RS:0;4585c39bed03:37381 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/oldWALs 2024-11-27T21:16:58,424 INFO [RS:0;4585c39bed03:37381 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4585c39bed03%2C37381%2C1732742022265.meta:.meta(num 1732742025906) 2024-11-27T21:16:58,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741833_1009 (size=23835119) 2024-11-27T21:16:58,427 DEBUG [RS:0;4585c39bed03:37381 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/oldWALs 2024-11-27T21:16:58,427 INFO [RS:0;4585c39bed03:37381 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4585c39bed03%2C37381%2C1732742022265:(num 1732742025435) 2024-11-27T21:16:58,427 DEBUG [RS:0;4585c39bed03:37381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T21:16:58,427 INFO [RS:0;4585c39bed03:37381 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T21:16:58,427 INFO [RS:0;4585c39bed03:37381 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T21:16:58,428 INFO [RS:0;4585c39bed03:37381 {}] hbase.ChoreService(370): Chore service for: regionserver/4585c39bed03:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T21:16:58,428 INFO [RS:0;4585c39bed03:37381 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T21:16:58,428 INFO [regionserver/4585c39bed03:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T21:16:58,429 INFO [RS:0;4585c39bed03:37381 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37381 2024-11-27T21:16:58,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T21:16:58,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4585c39bed03,37381,1732742022265 2024-11-27T21:16:58,432 INFO [RS:0;4585c39bed03:37381 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T21:16:58,433 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4585c39bed03,37381,1732742022265] 2024-11-27T21:16:58,437 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4585c39bed03,37381,1732742022265 already deleted, retry=false 2024-11-27T21:16:58,437 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4585c39bed03,37381,1732742022265 expired; onlineServers=0 2024-11-27T21:16:58,437 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4585c39bed03,38503,1732742021440' ***** 2024-11-27T21:16:58,437 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T21:16:58,437 INFO [M:0;4585c39bed03:38503 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T21:16:58,437 INFO [M:0;4585c39bed03:38503 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T21:16:58,437 DEBUG [M:0;4585c39bed03:38503 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T21:16:58,437 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T21:16:58,437 DEBUG [M:0;4585c39bed03:38503 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T21:16:58,437 DEBUG [master/4585c39bed03:0:becomeActiveMaster-HFileCleaner.small.0-1732742025157 {}] cleaner.HFileCleaner(306): Exit Thread[master/4585c39bed03:0:becomeActiveMaster-HFileCleaner.small.0-1732742025157,5,FailOnTimeoutGroup] 2024-11-27T21:16:58,437 DEBUG [master/4585c39bed03:0:becomeActiveMaster-HFileCleaner.large.0-1732742025153 {}] cleaner.HFileCleaner(306): Exit Thread[master/4585c39bed03:0:becomeActiveMaster-HFileCleaner.large.0-1732742025153,5,FailOnTimeoutGroup] 2024-11-27T21:16:58,437 INFO [M:0;4585c39bed03:38503 {}] hbase.ChoreService(370): Chore service for: master/4585c39bed03:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T21:16:58,438 INFO [M:0;4585c39bed03:38503 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T21:16:58,438 DEBUG [M:0;4585c39bed03:38503 {}] master.HMaster(1795): Stopping service threads 2024-11-27T21:16:58,438 INFO [M:0;4585c39bed03:38503 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T21:16:58,438 INFO [M:0;4585c39bed03:38503 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T21:16:58,438 ERROR [M:0;4585c39bed03:38503 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:42399 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:42399,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-27T21:16:58,439 INFO [M:0;4585c39bed03:38503 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T21:16:58,439 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T21:16:58,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T21:16:58,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T21:16:58,440 DEBUG [M:0;4585c39bed03:38503 {}] zookeeper.ZKUtil(347): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T21:16:58,440 WARN [M:0;4585c39bed03:38503 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T21:16:58,441 INFO [M:0;4585c39bed03:38503 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/.lastflushedseqids 2024-11-27T21:16:58,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742500_1676 (size=72) 2024-11-27T21:16:58,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T21:16:58,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37381-0x1015b46e86a0001, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T21:16:58,535 INFO [RS:0;4585c39bed03:37381 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T21:16:58,536 INFO [RS:0;4585c39bed03:37381 {}] regionserver.HRegionServer(1031): Exiting; stopping=4585c39bed03,37381,1732742022265; zookeeper connection closed. 2024-11-27T21:16:58,536 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@66a3d415 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@66a3d415 2024-11-27T21:16:58,536 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-27T21:16:58,850 INFO [M:0;4585c39bed03:38503 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T21:16:58,850 INFO [M:0;4585c39bed03:38503 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T21:16:58,850 DEBUG [M:0;4585c39bed03:38503 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T21:16:58,850 INFO [M:0;4585c39bed03:38503 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T21:16:58,850 DEBUG [M:0;4585c39bed03:38503 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T21:16:58,850 DEBUG [M:0;4585c39bed03:38503 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T21:16:58,850 DEBUG [M:0;4585c39bed03:38503 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T21:16:58,850 INFO [M:0;4585c39bed03:38503 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=810.87 KB heapSize=996.30 KB 2024-11-27T21:16:58,866 DEBUG [M:0;4585c39bed03:38503 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c1ea400909414de1a17a03f1a036c549 is 82, key is hbase:meta,,1/info:regioninfo/1732742026023/Put/seqid=0 2024-11-27T21:16:58,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742501_1677 (size=5672) 2024-11-27T21:16:59,269 INFO [M:0;4585c39bed03:38503 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2298 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c1ea400909414de1a17a03f1a036c549 2024-11-27T21:16:59,298 DEBUG [M:0;4585c39bed03:38503 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c76ca4737ba8409c939c40ec075c5e60 is 2479, key is \x00\x00\x00\x00\x00\x00\x00b/proc:d/1732742121617/Put/seqid=0 2024-11-27T21:16:59,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742502_1678 (size=46386) 2024-11-27T21:16:59,702 INFO [M:0;4585c39bed03:38503 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=810.31 KB at sequenceid=2298 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c76ca4737ba8409c939c40ec075c5e60 2024-11-27T21:16:59,706 INFO [M:0;4585c39bed03:38503 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c76ca4737ba8409c939c40ec075c5e60 2024-11-27T21:16:59,720 DEBUG [M:0;4585c39bed03:38503 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b949028f61b143ae8ca3958d9c2b1eec is 69, key is 4585c39bed03,37381,1732742022265/rs:state/1732742025175/Put/seqid=0 2024-11-27T21:16:59,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073742503_1679 (size=5156) 2024-11-27T21:17:00,124 INFO [M:0;4585c39bed03:38503 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2298 (bloomFilter=true), to=hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b949028f61b143ae8ca3958d9c2b1eec 2024-11-27T21:17:00,127 DEBUG [M:0;4585c39bed03:38503 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c1ea400909414de1a17a03f1a036c549 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c1ea400909414de1a17a03f1a036c549 2024-11-27T21:17:00,129 INFO [M:0;4585c39bed03:38503 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c1ea400909414de1a17a03f1a036c549, entries=8, sequenceid=2298, filesize=5.5 K 2024-11-27T21:17:00,130 DEBUG [M:0;4585c39bed03:38503 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c76ca4737ba8409c939c40ec075c5e60 as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c76ca4737ba8409c939c40ec075c5e60 2024-11-27T21:17:00,132 INFO [M:0;4585c39bed03:38503 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c76ca4737ba8409c939c40ec075c5e60 2024-11-27T21:17:00,133 INFO [M:0;4585c39bed03:38503 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c76ca4737ba8409c939c40ec075c5e60, entries=176, sequenceid=2298, filesize=45.3 K 2024-11-27T21:17:00,133 DEBUG [M:0;4585c39bed03:38503 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b949028f61b143ae8ca3958d9c2b1eec as hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b949028f61b143ae8ca3958d9c2b1eec 2024-11-27T21:17:00,135 INFO [M:0;4585c39bed03:38503 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42399/user/jenkins/test-data/06c08626-4a8b-e0b3-9c63-9e71f951b8de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b949028f61b143ae8ca3958d9c2b1eec, entries=1, sequenceid=2298, filesize=5.0 K 2024-11-27T21:17:00,136 INFO [M:0;4585c39bed03:38503 {}] regionserver.HRegion(3140): Finished flush of dataSize ~810.87 KB/830326, heapSize ~996.01 KB/1019912, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1286ms, sequenceid=2298, compaction requested=false 2024-11-27T21:17:00,137 INFO [M:0;4585c39bed03:38503 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T21:17:00,137 DEBUG [M:0;4585c39bed03:38503 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732742218850Disabling compacts and flushes for region at 1732742218850Disabling writes for close at 1732742218850Obtaining lock to block concurrent updates at 1732742218850Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732742218850Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=830326, getHeapSize=1020152, getOffHeapSize=0, getCellsCount=2499 at 1732742218851 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732742218851Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732742218851Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732742218865 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732742218865Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732742219272 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732742219297 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732742219297Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732742219706 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732742219720 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732742219720Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2226e22: reopening flushed file at 1732742220126 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4df2f980: reopening flushed file at 1732742220130 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ccdd3d3: reopening flushed file at 1732742220133 (+3 ms)Finished flush of dataSize ~810.87 KB/830326, heapSize ~996.01 KB/1019912, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1286ms, sequenceid=2298, compaction requested=false at 1732742220136 (+3 ms)Writing region close event to WAL at 1732742220137 (+1 ms)Closed at 1732742220137 2024-11-27T21:17:00,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40567 is added to blk_1073741830_1006 (size=980919) 2024-11-27T21:17:00,140 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T21:17:00,141 INFO [M:0;4585c39bed03:38503 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T21:17:00,141 INFO [M:0;4585c39bed03:38503 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38503 2024-11-27T21:17:00,141 INFO [M:0;4585c39bed03:38503 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T21:17:00,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T21:17:00,243 INFO [M:0;4585c39bed03:38503 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T21:17:00,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38503-0x1015b46e86a0000, quorum=127.0.0.1:60751, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T21:17:00,247 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T21:17:00,250 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T21:17:00,250 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T21:17:00,250 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T21:17:00,250 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/hadoop.log.dir/,STOPPED} 2024-11-27T21:17:00,253 WARN [BP-937257277-172.17.0.2-1732742017740 heartbeating to localhost/127.0.0.1:42399 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T21:17:00,253 WARN [BP-937257277-172.17.0.2-1732742017740 heartbeating to localhost/127.0.0.1:42399 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-937257277-172.17.0.2-1732742017740 (Datanode Uuid f92784c2-4a78-4fdf-8fac-aaadf4e44198) service to localhost/127.0.0.1:42399 2024-11-27T21:17:00,253 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T21:17:00,254 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T21:17:00,256 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/cluster_5e7b66de-1091-31c3-1f71-380834b464b9/data/data1/current/BP-937257277-172.17.0.2-1732742017740 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T21:17:00,256 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/cluster_5e7b66de-1091-31c3-1f71-380834b464b9/data/data2/current/BP-937257277-172.17.0.2-1732742017740 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T21:17:00,257 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T21:17:00,264 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T21:17:00,264 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T21:17:00,264 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T21:17:00,265 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T21:17:00,265 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a5520878-6c46-3d08-9c35-e0d9ae8cb609/hadoop.log.dir/,STOPPED} 2024-11-27T21:17:00,281 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T21:17:00,404 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down